2024-11-07 10:43:45,620 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-07 10:43:45,637 main DEBUG Took 0.015487 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-07 10:43:45,638 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-07 10:43:45,638 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-07 10:43:45,639 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-07 10:43:45,640 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 10:43:45,646 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-07 10:43:45,658 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 10:43:45,659 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 10:43:45,660 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 10:43:45,660 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 10:43:45,660 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 10:43:45,660 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 10:43:45,661 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 10:43:45,662 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 10:43:45,662 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 10:43:45,662 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 10:43:45,663 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 10:43:45,663 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 10:43:45,664 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 10:43:45,664 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 10:43:45,665 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 10:43:45,665 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 10:43:45,665 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 10:43:45,665 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 10:43:45,666 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 10:43:45,666 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 10:43:45,666 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 10:43:45,667 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 10:43:45,667 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 10:43:45,667 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-07 10:43:45,668 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 10:43:45,668 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-07 10:43:45,669 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-07 10:43:45,671 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-07 10:43:45,672 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-07 10:43:45,673 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-07 10:43:45,674 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-07 10:43:45,674 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-07 10:43:45,683 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-07 10:43:45,685 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-07 10:43:45,686 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-07 10:43:45,687 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-07 10:43:45,687 main DEBUG createAppenders(={Console}) 2024-11-07 10:43:45,688 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-07 10:43:45,688 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-07 10:43:45,688 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-07 10:43:45,689 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-07 10:43:45,689 main DEBUG OutputStream closed 2024-11-07 10:43:45,689 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-07 10:43:45,689 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-07 10:43:45,689 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-07 10:43:45,754 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-07 10:43:45,755 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-07 10:43:45,756 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-07 10:43:45,757 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-07 10:43:45,758 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-07 10:43:45,758 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-07 10:43:45,758 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-07 10:43:45,759 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-07 10:43:45,759 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-07 10:43:45,759 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-07 10:43:45,760 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-07 10:43:45,760 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-07 10:43:45,760 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-07 10:43:45,760 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-07 10:43:45,761 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-07 10:43:45,761 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-07 10:43:45,761 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-07 10:43:45,762 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-07 10:43:45,764 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07 10:43:45,764 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-07 10:43:45,765 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-07 10:43:45,765 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-07T10:43:45,779 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta timeout: 13 mins 2024-11-07 10:43:45,782 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-07 10:43:45,782 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-07T10:43:45,815 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.procedure.TestSCPWithMeta#testRecoveryAndDoubleExecutionOnRsWithMeta Thread=11, OpenFileDescriptor=285, MaxFileDescriptor=1048576, SystemLoadAverage=330, ProcessCount=11, AvailableMemoryMB=5114 2024-11-07T10:43:46,064 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51 2024-11-07T10:43:46,065 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-07T10:43:46,079 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/cluster_1da8066b-c576-62ef-93bb-fd1753eba61c, deleteOnExit=true 2024-11-07T10:43:46,079 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-07T10:43:46,080 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/test.cache.data in system properties and HBase conf 2024-11-07T10:43:46,081 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/hadoop.tmp.dir in system properties and HBase conf 2024-11-07T10:43:46,081 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/hadoop.log.dir in system properties and HBase conf 2024-11-07T10:43:46,082 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-07T10:43:46,082 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-07T10:43:46,082 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-07T10:43:46,161 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-07T10:43:46,244 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-07T10:43:46,248 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-07T10:43:46,249 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-07T10:43:46,250 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-07T10:43:46,250 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-07T10:43:46,251 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-07T10:43:46,252 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-07T10:43:46,252 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-07T10:43:46,253 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-07T10:43:46,253 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-07T10:43:46,254 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/nfs.dump.dir in system properties and HBase conf 2024-11-07T10:43:46,255 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/java.io.tmpdir in system properties and HBase conf 2024-11-07T10:43:46,255 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-07T10:43:46,256 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-07T10:43:46,256 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-07T10:43:47,069 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-07T10:43:47,155 INFO [Time-limited test {}] log.Log(170): Logging initialized @2201ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-07T10:43:47,236 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T10:43:47,300 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T10:43:47,326 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T10:43:47,326 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T10:43:47,328 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-07T10:43:47,345 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T10:43:47,349 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2c447bb0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/hadoop.log.dir/,AVAILABLE} 2024-11-07T10:43:47,350 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4974766f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T10:43:47,548 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5183f126{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/java.io.tmpdir/jetty-localhost-43191-hadoop-hdfs-3_4_1-tests_jar-_-any-7068589907507768351/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-07T10:43:47,558 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4350931a{HTTP/1.1, (http/1.1)}{localhost:43191} 2024-11-07T10:43:47,558 INFO [Time-limited test {}] server.Server(415): Started @2605ms 2024-11-07T10:43:47,926 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T10:43:47,933 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T10:43:47,934 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T10:43:47,934 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T10:43:47,935 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-07T10:43:47,935 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93e0721{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/hadoop.log.dir/,AVAILABLE} 2024-11-07T10:43:47,936 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7da31712{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T10:43:48,055 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1d8160fd{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/java.io.tmpdir/jetty-localhost-42987-hadoop-hdfs-3_4_1-tests_jar-_-any-14789585452128580526/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T10:43:48,056 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7e504529{HTTP/1.1, (http/1.1)}{localhost:42987} 2024-11-07T10:43:48,056 INFO [Time-limited test {}] server.Server(415): Started @3103ms 2024-11-07T10:43:48,113 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-07T10:43:48,233 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T10:43:48,240 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T10:43:48,241 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T10:43:48,241 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T10:43:48,242 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-07T10:43:48,243 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7186ce9f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/hadoop.log.dir/,AVAILABLE} 2024-11-07T10:43:48,243 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7f4017e5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T10:43:48,385 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@43a0d547{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/java.io.tmpdir/jetty-localhost-35313-hadoop-hdfs-3_4_1-tests_jar-_-any-18086486911241617119/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T10:43:48,386 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7b1b9e11{HTTP/1.1, (http/1.1)}{localhost:35313} 2024-11-07T10:43:48,386 INFO [Time-limited test {}] server.Server(415): Started @3433ms 2024-11-07T10:43:48,389 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-07T10:43:48,428 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-07T10:43:48,434 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-07T10:43:48,437 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-07T10:43:48,437 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-07T10:43:48,437 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-07T10:43:48,439 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7f9dc0ae{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/hadoop.log.dir/,AVAILABLE} 2024-11-07T10:43:48,440 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@30067af9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-07T10:43:48,535 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/cluster_1da8066b-c576-62ef-93bb-fd1753eba61c/data/data3/current/BP-1776563886-172.17.0.2-1730976226828/current, will proceed with Du for space computation calculation, 2024-11-07T10:43:48,535 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/cluster_1da8066b-c576-62ef-93bb-fd1753eba61c/data/data2/current/BP-1776563886-172.17.0.2-1730976226828/current, will proceed with Du for space computation calculation, 2024-11-07T10:43:48,535 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/cluster_1da8066b-c576-62ef-93bb-fd1753eba61c/data/data1/current/BP-1776563886-172.17.0.2-1730976226828/current, will proceed with Du for space computation calculation, 2024-11-07T10:43:48,535 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/cluster_1da8066b-c576-62ef-93bb-fd1753eba61c/data/data4/current/BP-1776563886-172.17.0.2-1730976226828/current, will proceed with Du for space computation calculation, 2024-11-07T10:43:48,573 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4d4e6c58{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/java.io.tmpdir/jetty-localhost-36047-hadoop-hdfs-3_4_1-tests_jar-_-any-11949437960433762083/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T10:43:48,574 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@57f81b0{HTTP/1.1, (http/1.1)}{localhost:36047} 2024-11-07T10:43:48,574 INFO [Time-limited test {}] server.Server(415): Started @3621ms 2024-11-07T10:43:48,576 WARN [Thread-82 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-07T10:43:48,576 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-07T10:43:48,577 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-07T10:43:48,644 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x28b25cb831f9f813 with lease ID 0xc7e2da3d77570423: Processing first storage report for DS-5d0e97fa-7109-4597-9d71-8fb432e0463c from datanode DatanodeRegistration(127.0.0.1:40045, datanodeUuid=107098e0-f1c7-4f05-8e1d-4b90535e9624, infoPort=41569, infoSecurePort=0, ipcPort=38893, storageInfo=lv=-57;cid=testClusterID;nsid=1622631827;c=1730976226828) 2024-11-07T10:43:48,645 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x28b25cb831f9f813 with lease ID 0xc7e2da3d77570423: from storage DS-5d0e97fa-7109-4597-9d71-8fb432e0463c node DatanodeRegistration(127.0.0.1:40045, datanodeUuid=107098e0-f1c7-4f05-8e1d-4b90535e9624, infoPort=41569, infoSecurePort=0, ipcPort=38893, storageInfo=lv=-57;cid=testClusterID;nsid=1622631827;c=1730976226828), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-07T10:43:48,646 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x16ccecf62823a4da with lease ID 0xc7e2da3d77570424: Processing first storage report for DS-4666ca76-3f5e-48c3-8d05-ac47a18a697b from datanode DatanodeRegistration(127.0.0.1:41673, datanodeUuid=07928241-511d-4e49-af54-949e15b4e3e7, infoPort=33799, infoSecurePort=0, ipcPort=39321, storageInfo=lv=-57;cid=testClusterID;nsid=1622631827;c=1730976226828) 2024-11-07T10:43:48,646 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x16ccecf62823a4da with lease ID 0xc7e2da3d77570424: from storage DS-4666ca76-3f5e-48c3-8d05-ac47a18a697b node DatanodeRegistration(127.0.0.1:41673, datanodeUuid=07928241-511d-4e49-af54-949e15b4e3e7, infoPort=33799, infoSecurePort=0, ipcPort=39321, storageInfo=lv=-57;cid=testClusterID;nsid=1622631827;c=1730976226828), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-07T10:43:48,646 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x28b25cb831f9f813 with lease ID 0xc7e2da3d77570423: Processing first storage report for DS-8bf12d82-edc9-4437-96ce-c5326302820e from datanode DatanodeRegistration(127.0.0.1:40045, datanodeUuid=107098e0-f1c7-4f05-8e1d-4b90535e9624, infoPort=41569, infoSecurePort=0, ipcPort=38893, storageInfo=lv=-57;cid=testClusterID;nsid=1622631827;c=1730976226828) 2024-11-07T10:43:48,647 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x28b25cb831f9f813 with lease ID 0xc7e2da3d77570423: from storage DS-8bf12d82-edc9-4437-96ce-c5326302820e node DatanodeRegistration(127.0.0.1:40045, datanodeUuid=107098e0-f1c7-4f05-8e1d-4b90535e9624, infoPort=41569, infoSecurePort=0, ipcPort=38893, storageInfo=lv=-57;cid=testClusterID;nsid=1622631827;c=1730976226828), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-07T10:43:48,647 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x16ccecf62823a4da with lease ID 0xc7e2da3d77570424: Processing first storage report for DS-7faba07a-6f22-4db8-b701-5271a8175049 from datanode DatanodeRegistration(127.0.0.1:41673, datanodeUuid=07928241-511d-4e49-af54-949e15b4e3e7, infoPort=33799, infoSecurePort=0, ipcPort=39321, storageInfo=lv=-57;cid=testClusterID;nsid=1622631827;c=1730976226828) 2024-11-07T10:43:48,647 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x16ccecf62823a4da with lease ID 0xc7e2da3d77570424: from storage DS-7faba07a-6f22-4db8-b701-5271a8175049 node DatanodeRegistration(127.0.0.1:41673, datanodeUuid=07928241-511d-4e49-af54-949e15b4e3e7, infoPort=33799, infoSecurePort=0, ipcPort=39321, storageInfo=lv=-57;cid=testClusterID;nsid=1622631827;c=1730976226828), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-07T10:43:48,699 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/cluster_1da8066b-c576-62ef-93bb-fd1753eba61c/data/data5/current/BP-1776563886-172.17.0.2-1730976226828/current, will proceed with Du for space computation calculation, 2024-11-07T10:43:48,699 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/cluster_1da8066b-c576-62ef-93bb-fd1753eba61c/data/data6/current/BP-1776563886-172.17.0.2-1730976226828/current, will proceed with Du for space computation calculation, 2024-11-07T10:43:48,731 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-07T10:43:48,736 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9c138daf72916539 with lease ID 0xc7e2da3d77570425: Processing first storage report for DS-ca0d026c-750a-4349-8e6f-ce26c9f0979a from datanode DatanodeRegistration(127.0.0.1:33357, datanodeUuid=7110ba11-93cf-48e7-b579-b2e0d9f18deb, infoPort=34291, infoSecurePort=0, ipcPort=46681, storageInfo=lv=-57;cid=testClusterID;nsid=1622631827;c=1730976226828) 2024-11-07T10:43:48,736 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9c138daf72916539 with lease ID 0xc7e2da3d77570425: from storage DS-ca0d026c-750a-4349-8e6f-ce26c9f0979a node DatanodeRegistration(127.0.0.1:33357, datanodeUuid=7110ba11-93cf-48e7-b579-b2e0d9f18deb, infoPort=34291, infoSecurePort=0, ipcPort=46681, storageInfo=lv=-57;cid=testClusterID;nsid=1622631827;c=1730976226828), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-07T10:43:48,737 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9c138daf72916539 with lease ID 0xc7e2da3d77570425: Processing first storage report for DS-c9e2ae64-3ac5-43d0-aba0-9241bc1e8f89 from datanode DatanodeRegistration(127.0.0.1:33357, datanodeUuid=7110ba11-93cf-48e7-b579-b2e0d9f18deb, infoPort=34291, infoSecurePort=0, ipcPort=46681, storageInfo=lv=-57;cid=testClusterID;nsid=1622631827;c=1730976226828) 2024-11-07T10:43:48,737 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9c138daf72916539 with lease ID 0xc7e2da3d77570425: from storage DS-c9e2ae64-3ac5-43d0-aba0-9241bc1e8f89 node DatanodeRegistration(127.0.0.1:33357, datanodeUuid=7110ba11-93cf-48e7-b579-b2e0d9f18deb, infoPort=34291, infoSecurePort=0, ipcPort=46681, storageInfo=lv=-57;cid=testClusterID;nsid=1622631827;c=1730976226828), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-07T10:43:48,929 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51 2024-11-07T10:43:49,003 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/cluster_1da8066b-c576-62ef-93bb-fd1753eba61c/zookeeper_0, clientPort=59788, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/cluster_1da8066b-c576-62ef-93bb-fd1753eba61c/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/cluster_1da8066b-c576-62ef-93bb-fd1753eba61c/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-07T10:43:49,013 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59788 2024-11-07T10:43:49,023 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T10:43:49,026 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T10:43:49,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741825_1001 (size=7) 2024-11-07T10:43:49,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741825_1001 (size=7) 2024-11-07T10:43:49,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741825_1001 (size=7) 2024-11-07T10:43:49,662 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 with version=8 2024-11-07T10:43:49,663 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/hbase-staging 2024-11-07T10:43:49,757 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-07T10:43:50,006 INFO [Time-limited test {}] client.ConnectionUtils(128): master/10641b2c0a69:0 server-side Connection retries=45 2024-11-07T10:43:50,017 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T10:43:50,017 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T10:43:50,022 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T10:43:50,022 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T10:43:50,022 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T10:43:50,155 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-07T10:43:50,215 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-07T10:43:50,223 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-07T10:43:50,227 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T10:43:50,255 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 7030 (auto-detected) 2024-11-07T10:43:50,256 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-07T10:43:50,275 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43605 2024-11-07T10:43:50,296 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43605 connecting to ZooKeeper ensemble=127.0.0.1:59788 2024-11-07T10:43:50,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:436050x0, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T10:43:50,330 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43605-0x10019d731180000 connected 2024-11-07T10:43:50,356 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T10:43:50,359 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T10:43:50,369 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T10:43:50,373 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393, hbase.cluster.distributed=false 2024-11-07T10:43:50,396 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T10:43:50,400 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43605 2024-11-07T10:43:50,401 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43605 2024-11-07T10:43:50,401 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43605 2024-11-07T10:43:50,402 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43605 2024-11-07T10:43:50,402 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43605 2024-11-07T10:43:50,515 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/10641b2c0a69:0 server-side Connection retries=45 2024-11-07T10:43:50,517 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T10:43:50,517 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T10:43:50,518 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T10:43:50,518 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T10:43:50,518 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T10:43:50,521 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T10:43:50,524 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T10:43:50,525 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40573 2024-11-07T10:43:50,528 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40573 connecting to ZooKeeper ensemble=127.0.0.1:59788 2024-11-07T10:43:50,529 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T10:43:50,533 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T10:43:50,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:405730x0, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T10:43:50,539 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40573-0x10019d731180001 connected 2024-11-07T10:43:50,539 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T10:43:50,544 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-07T10:43:50,551 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-07T10:43:50,554 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T10:43:50,559 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T10:43:50,559 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40573 2024-11-07T10:43:50,560 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40573 2024-11-07T10:43:50,560 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40573 2024-11-07T10:43:50,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40573 2024-11-07T10:43:50,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40573 2024-11-07T10:43:50,579 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/10641b2c0a69:0 server-side Connection retries=45 2024-11-07T10:43:50,579 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T10:43:50,579 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T10:43:50,580 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T10:43:50,580 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T10:43:50,580 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T10:43:50,580 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T10:43:50,581 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T10:43:50,582 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41401 2024-11-07T10:43:50,583 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41401 connecting to ZooKeeper ensemble=127.0.0.1:59788 2024-11-07T10:43:50,584 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T10:43:50,586 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T10:43:50,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:414010x0, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T10:43:50,592 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41401-0x10019d731180002 connected 2024-11-07T10:43:50,592 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41401-0x10019d731180002, quorum=127.0.0.1:59788, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T10:43:50,593 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-07T10:43:50,594 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-07T10:43:50,595 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41401-0x10019d731180002, quorum=127.0.0.1:59788, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T10:43:50,597 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41401-0x10019d731180002, quorum=127.0.0.1:59788, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T10:43:50,599 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41401 2024-11-07T10:43:50,599 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41401 2024-11-07T10:43:50,599 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41401 2024-11-07T10:43:50,600 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41401 2024-11-07T10:43:50,601 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41401 2024-11-07T10:43:50,620 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/10641b2c0a69:0 server-side Connection retries=45 2024-11-07T10:43:50,620 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T10:43:50,621 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-07T10:43:50,621 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-07T10:43:50,621 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-07T10:43:50,621 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-07T10:43:50,622 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-07T10:43:50,622 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-07T10:43:50,623 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39013 2024-11-07T10:43:50,624 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39013 connecting to ZooKeeper ensemble=127.0.0.1:59788 2024-11-07T10:43:50,625 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T10:43:50,628 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T10:43:50,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:390130x0, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-07T10:43:50,634 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:390130x0, quorum=127.0.0.1:59788, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T10:43:50,634 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39013-0x10019d731180003 connected 2024-11-07T10:43:50,635 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-07T10:43:50,635 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-07T10:43:50,636 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-07T10:43:50,638 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-07T10:43:50,639 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39013 2024-11-07T10:43:50,640 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39013 2024-11-07T10:43:50,640 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39013 2024-11-07T10:43:50,641 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39013 2024-11-07T10:43:50,641 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39013 2024-11-07T10:43:50,655 DEBUG [M:0;10641b2c0a69:43605 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;10641b2c0a69:43605 2024-11-07T10:43:50,656 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/10641b2c0a69,43605,1730976229810 2024-11-07T10:43:50,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T10:43:50,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T10:43:50,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41401-0x10019d731180002, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T10:43:50,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T10:43:50,666 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/10641b2c0a69,43605,1730976229810 2024-11-07T10:43:50,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41401-0x10019d731180002, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T10:43:50,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T10:43:50,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-07T10:43:50,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:43:50,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41401-0x10019d731180002, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:43:50,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:43:50,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:43:50,689 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-07T10:43:50,690 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/10641b2c0a69,43605,1730976229810 from backup master directory 2024-11-07T10:43:50,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/10641b2c0a69,43605,1730976229810 2024-11-07T10:43:50,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41401-0x10019d731180002, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T10:43:50,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T10:43:50,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T10:43:50,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-07T10:43:50,694 WARN [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T10:43:50,694 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=10641b2c0a69,43605,1730976229810 2024-11-07T10:43:50,696 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-07T10:43:50,698 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-07T10:43:50,753 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/hbase.id] with ID: 81cac59c-8db3-42bf-8d2a-6a03e22005b3 2024-11-07T10:43:50,754 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/.tmp/hbase.id 2024-11-07T10:43:50,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741826_1002 (size=42) 2024-11-07T10:43:50,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741826_1002 (size=42) 2024-11-07T10:43:50,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741826_1002 (size=42) 2024-11-07T10:43:50,769 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/.tmp/hbase.id]:[hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/hbase.id] 2024-11-07T10:43:50,814 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-07T10:43:50,818 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-07T10:43:50,838 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-07T10:43:50,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:43:50,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:43:50,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:43:50,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41401-0x10019d731180002, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:43:50,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741827_1003 (size=196) 2024-11-07T10:43:50,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741827_1003 (size=196) 2024-11-07T10:43:50,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741827_1003 (size=196) 2024-11-07T10:43:50,878 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T10:43:50,880 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-07T10:43:50,897 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T10:43:50,902 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T10:43:50,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741828_1004 (size=1189) 2024-11-07T10:43:50,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741828_1004 (size=1189) 2024-11-07T10:43:50,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741828_1004 (size=1189) 2024-11-07T10:43:50,956 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/MasterData/data/master/store 2024-11-07T10:43:50,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741829_1005 (size=34) 2024-11-07T10:43:50,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741829_1005 (size=34) 2024-11-07T10:43:50,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741829_1005 (size=34) 2024-11-07T10:43:50,984 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-07T10:43:50,987 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:50,988 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T10:43:50,988 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T10:43:50,989 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T10:43:50,990 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T10:43:50,990 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T10:43:50,990 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T10:43:50,991 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1730976230988Disabling compacts and flushes for region at 1730976230988Disabling writes for close at 1730976230990 (+2 ms)Writing region close event to WAL at 1730976230990Closed at 1730976230990 2024-11-07T10:43:50,993 WARN [master/10641b2c0a69:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/MasterData/data/master/store/.initializing 2024-11-07T10:43:50,994 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/MasterData/WALs/10641b2c0a69,43605,1730976229810 2024-11-07T10:43:51,002 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T10:43:51,016 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=10641b2c0a69%2C43605%2C1730976229810, suffix=, logDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/MasterData/WALs/10641b2c0a69,43605,1730976229810, archiveDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/MasterData/oldWALs, maxLogs=10 2024-11-07T10:43:51,039 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/MasterData/WALs/10641b2c0a69,43605,1730976229810/10641b2c0a69%2C43605%2C1730976229810.1730976231020, exclude list is [], retry=0 2024-11-07T10:43:51,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40045,DS-5d0e97fa-7109-4597-9d71-8fb432e0463c,DISK] 2024-11-07T10:43:51,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41673,DS-4666ca76-3f5e-48c3-8d05-ac47a18a697b,DISK] 2024-11-07T10:43:51,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33357,DS-ca0d026c-750a-4349-8e6f-ce26c9f0979a,DISK] 2024-11-07T10:43:51,062 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-07T10:43:51,101 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/MasterData/WALs/10641b2c0a69,43605,1730976229810/10641b2c0a69%2C43605%2C1730976229810.1730976231020 2024-11-07T10:43:51,102 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41569:41569),(127.0.0.1/127.0.0.1:33799:33799),(127.0.0.1/127.0.0.1:34291:34291)] 2024-11-07T10:43:51,103 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-07T10:43:51,103 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:51,106 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:43:51,107 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:43:51,145 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:43:51,171 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-07T10:43:51,175 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:51,178 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:43:51,179 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:43:51,182 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-07T10:43:51,182 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:51,183 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:51,183 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:43:51,186 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-07T10:43:51,186 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:51,187 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:51,187 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:43:51,190 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-07T10:43:51,190 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:51,191 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:51,191 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:43:51,194 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:43:51,195 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:43:51,199 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:43:51,200 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:43:51,203 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T10:43:51,207 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-07T10:43:51,211 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:43:51,212 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63636485, jitterRate=-0.05174247920513153}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T10:43:51,218 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1730976231120Initializing all the Stores at 1730976231122 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730976231122Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976231123 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976231123Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976231123Cleaning up temporary data from old regions at 1730976231200 (+77 ms)Region opened successfully at 1730976231218 (+18 ms) 2024-11-07T10:43:51,219 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-07T10:43:51,253 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7418301b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=10641b2c0a69/172.17.0.2:0 2024-11-07T10:43:51,286 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-07T10:43:51,298 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T10:43:51,298 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T10:43:51,301 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T10:43:51,302 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-07T10:43:51,307 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-07T10:43:51,307 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-07T10:43:51,331 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-07T10:43:51,341 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-07T10:43:51,344 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-07T10:43:51,347 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-07T10:43:51,348 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-07T10:43:51,350 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-07T10:43:51,352 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-07T10:43:51,356 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-07T10:43:51,357 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-07T10:43:51,358 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-07T10:43:51,360 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-07T10:43:51,377 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-07T10:43:51,378 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-07T10:43:51,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T10:43:51,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T10:43:51,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41401-0x10019d731180002, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T10:43:51,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:43:51,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-07T10:43:51,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:43:51,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41401-0x10019d731180002, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:43:51,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:43:51,385 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=10641b2c0a69,43605,1730976229810, sessionid=0x10019d731180000, setting cluster-up flag (Was=false) 2024-11-07T10:43:51,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:43:51,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:43:51,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41401-0x10019d731180002, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:43:51,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:43:51,402 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-07T10:43:51,404 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=10641b2c0a69,43605,1730976229810 2024-11-07T10:43:51,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:43:51,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:43:51,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41401-0x10019d731180002, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:43:51,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:43:51,414 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-07T10:43:51,416 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=10641b2c0a69,43605,1730976229810 2024-11-07T10:43:51,422 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-07T10:43:51,445 INFO [RS:1;10641b2c0a69:41401 {}] regionserver.HRegionServer(746): ClusterId : 81cac59c-8db3-42bf-8d2a-6a03e22005b3 2024-11-07T10:43:51,445 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(746): ClusterId : 81cac59c-8db3-42bf-8d2a-6a03e22005b3 2024-11-07T10:43:51,445 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(746): ClusterId : 81cac59c-8db3-42bf-8d2a-6a03e22005b3 2024-11-07T10:43:51,447 DEBUG [RS:1;10641b2c0a69:41401 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-07T10:43:51,447 DEBUG [RS:2;10641b2c0a69:39013 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-07T10:43:51,447 DEBUG [RS:0;10641b2c0a69:40573 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-07T10:43:51,452 DEBUG [RS:0;10641b2c0a69:40573 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-07T10:43:51,452 DEBUG [RS:2;10641b2c0a69:39013 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-07T10:43:51,452 DEBUG [RS:1;10641b2c0a69:41401 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-07T10:43:51,452 DEBUG [RS:0;10641b2c0a69:40573 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-07T10:43:51,452 DEBUG [RS:2;10641b2c0a69:39013 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-07T10:43:51,452 DEBUG [RS:1;10641b2c0a69:41401 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-07T10:43:51,456 DEBUG [RS:2;10641b2c0a69:39013 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-07T10:43:51,456 DEBUG [RS:0;10641b2c0a69:40573 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-07T10:43:51,456 DEBUG [RS:1;10641b2c0a69:41401 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-07T10:43:51,456 DEBUG [RS:0;10641b2c0a69:40573 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a4d4f7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=10641b2c0a69/172.17.0.2:0 2024-11-07T10:43:51,456 DEBUG [RS:1;10641b2c0a69:41401 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54669eb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=10641b2c0a69/172.17.0.2:0 2024-11-07T10:43:51,456 DEBUG [RS:2;10641b2c0a69:39013 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dbd341e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=10641b2c0a69/172.17.0.2:0 2024-11-07T10:43:51,476 DEBUG [RS:1;10641b2c0a69:41401 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;10641b2c0a69:41401 2024-11-07T10:43:51,477 DEBUG [RS:2;10641b2c0a69:39013 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;10641b2c0a69:39013 2024-11-07T10:43:51,477 DEBUG [RS:0;10641b2c0a69:40573 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;10641b2c0a69:40573 2024-11-07T10:43:51,478 INFO [RS:1;10641b2c0a69:41401 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-07T10:43:51,479 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-07T10:43:51,479 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-07T10:43:51,479 INFO [RS:1;10641b2c0a69:41401 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-07T10:43:51,479 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-07T10:43:51,479 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-07T10:43:51,479 DEBUG [RS:1;10641b2c0a69:41401 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-07T10:43:51,479 DEBUG [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-07T10:43:51,479 DEBUG [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-07T10:43:51,481 INFO [RS:1;10641b2c0a69:41401 {}] regionserver.HRegionServer(2659): reportForDuty to master=10641b2c0a69,43605,1730976229810 with port=41401, startcode=1730976230578 2024-11-07T10:43:51,481 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(2659): reportForDuty to master=10641b2c0a69,43605,1730976229810 with port=39013, startcode=1730976230620 2024-11-07T10:43:51,481 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(2659): reportForDuty to master=10641b2c0a69,43605,1730976229810 with port=40573, startcode=1730976230476 2024-11-07T10:43:51,494 DEBUG [RS:2;10641b2c0a69:39013 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T10:43:51,494 DEBUG [RS:1;10641b2c0a69:41401 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T10:43:51,494 DEBUG [RS:0;10641b2c0a69:40573 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-07T10:43:51,507 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T10:43:51,518 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-07T10:43:51,529 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-07T10:43:51,533 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38881, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T10:43:51,533 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52941, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T10:43:51,533 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33991, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-07T10:43:51,537 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 10641b2c0a69,43605,1730976229810 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-07T10:43:51,541 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43605 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T10:43:51,545 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/10641b2c0a69:0, corePoolSize=5, maxPoolSize=5 2024-11-07T10:43:51,546 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/10641b2c0a69:0, corePoolSize=5, maxPoolSize=5 2024-11-07T10:43:51,546 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/10641b2c0a69:0, corePoolSize=5, maxPoolSize=5 2024-11-07T10:43:51,546 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/10641b2c0a69:0, corePoolSize=5, maxPoolSize=5 2024-11-07T10:43:51,547 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/10641b2c0a69:0, corePoolSize=10, maxPoolSize=10 2024-11-07T10:43:51,547 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,547 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/10641b2c0a69:0, corePoolSize=2, maxPoolSize=2 2024-11-07T10:43:51,547 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,548 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43605 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T10:43:51,549 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43605 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T10:43:51,550 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730976261550 2024-11-07T10:43:51,551 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-07T10:43:51,552 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-07T10:43:51,556 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T10:43:51,556 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-07T10:43:51,557 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-07T10:43:51,558 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-07T10:43:51,558 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-07T10:43:51,558 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-07T10:43:51,559 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,563 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-07T10:43:51,563 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:51,563 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-07T10:43:51,564 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-07T10:43:51,565 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-07T10:43:51,567 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-07T10:43:51,567 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-07T10:43:51,569 DEBUG [RS:1;10641b2c0a69:41401 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-07T10:43:51,569 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/10641b2c0a69:0:becomeActiveMaster-HFileCleaner.large.0-1730976231568,5,FailOnTimeoutGroup] 2024-11-07T10:43:51,569 DEBUG [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-07T10:43:51,569 DEBUG [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-07T10:43:51,570 WARN [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-07T10:43:51,570 WARN [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-07T10:43:51,570 WARN [RS:1;10641b2c0a69:41401 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-07T10:43:51,570 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/10641b2c0a69:0:becomeActiveMaster-HFileCleaner.small.0-1730976231570,5,FailOnTimeoutGroup] 2024-11-07T10:43:51,570 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,571 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-07T10:43:51,572 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,572 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741831_1007 (size=1321) 2024-11-07T10:43:51,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741831_1007 (size=1321) 2024-11-07T10:43:51,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741831_1007 (size=1321) 2024-11-07T10:43:51,634 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-07T10:43:51,634 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 2024-11-07T10:43:51,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741832_1008 (size=32) 2024-11-07T10:43:51,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741832_1008 (size=32) 2024-11-07T10:43:51,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741832_1008 (size=32) 2024-11-07T10:43:51,649 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:51,652 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T10:43:51,654 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T10:43:51,654 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:51,655 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:43:51,656 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-07T10:43:51,658 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T10:43:51,658 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:51,659 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:43:51,660 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T10:43:51,662 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T10:43:51,662 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:51,663 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:43:51,663 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T10:43:51,666 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T10:43:51,666 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:51,667 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:43:51,667 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T10:43:51,668 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740 2024-11-07T10:43:51,669 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740 2024-11-07T10:43:51,671 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(2659): reportForDuty to master=10641b2c0a69,43605,1730976229810 with port=40573, startcode=1730976230476 2024-11-07T10:43:51,671 INFO [RS:1;10641b2c0a69:41401 {}] regionserver.HRegionServer(2659): reportForDuty to master=10641b2c0a69,43605,1730976229810 with port=41401, startcode=1730976230578 2024-11-07T10:43:51,671 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(2659): reportForDuty to master=10641b2c0a69,43605,1730976229810 with port=39013, startcode=1730976230620 2024-11-07T10:43:51,672 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T10:43:51,672 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T10:43:51,674 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43605 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 10641b2c0a69,41401,1730976230578 2024-11-07T10:43:51,674 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T10:43:51,676 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T10:43:51,677 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43605 {}] master.ServerManager(517): Registering regionserver=10641b2c0a69,41401,1730976230578 2024-11-07T10:43:51,681 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:43:51,682 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60503040, jitterRate=-0.0984344482421875}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T10:43:51,686 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43605 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 10641b2c0a69,40573,1730976230476 2024-11-07T10:43:51,686 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1730976231650Initializing all the Stores at 1730976231651 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730976231651Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730976231651Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976231651Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730976231652 (+1 ms)Cleaning up temporary data from old regions at 1730976231672 (+20 ms)Region opened successfully at 1730976231686 (+14 ms) 2024-11-07T10:43:51,686 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43605 {}] master.ServerManager(517): Registering regionserver=10641b2c0a69,40573,1730976230476 2024-11-07T10:43:51,686 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-07T10:43:51,686 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-07T10:43:51,686 DEBUG [RS:1;10641b2c0a69:41401 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 2024-11-07T10:43:51,686 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-07T10:43:51,686 DEBUG [RS:1;10641b2c0a69:41401 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39973 2024-11-07T10:43:51,686 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T10:43:51,686 DEBUG [RS:1;10641b2c0a69:41401 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-07T10:43:51,686 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T10:43:51,688 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-07T10:43:51,688 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1730976231686Disabling compacts and flushes for region at 1730976231686Disabling writes for close at 1730976231686Writing region close event to WAL at 1730976231687 (+1 ms)Closed at 1730976231688 (+1 ms) 2024-11-07T10:43:51,691 DEBUG [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 2024-11-07T10:43:51,691 DEBUG [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39973 2024-11-07T10:43:51,691 DEBUG [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-07T10:43:51,691 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43605 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 10641b2c0a69,39013,1730976230620 2024-11-07T10:43:51,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T10:43:51,691 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43605 {}] master.ServerManager(517): Registering regionserver=10641b2c0a69,39013,1730976230620 2024-11-07T10:43:51,691 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T10:43:51,691 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-07T10:43:51,696 DEBUG [RS:1;10641b2c0a69:41401 {}] zookeeper.ZKUtil(111): regionserver:41401-0x10019d731180002, quorum=127.0.0.1:59788, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/10641b2c0a69,41401,1730976230578 2024-11-07T10:43:51,697 WARN [RS:1;10641b2c0a69:41401 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T10:43:51,697 INFO [RS:1;10641b2c0a69:41401 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T10:43:51,697 DEBUG [RS:0;10641b2c0a69:40573 {}] zookeeper.ZKUtil(111): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/10641b2c0a69,40573,1730976230476 2024-11-07T10:43:51,697 DEBUG [RS:1;10641b2c0a69:41401 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578 2024-11-07T10:43:51,697 WARN [RS:0;10641b2c0a69:40573 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T10:43:51,697 INFO [RS:0;10641b2c0a69:40573 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T10:43:51,698 DEBUG [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,40573,1730976230476 2024-11-07T10:43:51,699 DEBUG [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 2024-11-07T10:43:51,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-07T10:43:51,699 DEBUG [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39973 2024-11-07T10:43:51,699 DEBUG [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-07T10:43:51,701 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [10641b2c0a69,41401,1730976230578] 2024-11-07T10:43:51,701 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [10641b2c0a69,40573,1730976230476] 2024-11-07T10:43:51,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T10:43:51,703 DEBUG [RS:2;10641b2c0a69:39013 {}] zookeeper.ZKUtil(111): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/10641b2c0a69,39013,1730976230620 2024-11-07T10:43:51,703 WARN [RS:2;10641b2c0a69:39013 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-07T10:43:51,703 INFO [RS:2;10641b2c0a69:39013 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T10:43:51,703 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [10641b2c0a69,39013,1730976230620] 2024-11-07T10:43:51,703 DEBUG [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,39013,1730976230620 2024-11-07T10:43:51,718 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T10:43:51,721 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-07T10:43:51,728 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-07T10:43:51,728 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-07T10:43:51,728 INFO [RS:1;10641b2c0a69:41401 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-07T10:43:51,741 INFO [RS:1;10641b2c0a69:41401 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-07T10:43:51,741 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-07T10:43:51,741 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-07T10:43:51,747 INFO [RS:2;10641b2c0a69:39013 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-07T10:43:51,747 INFO [RS:0;10641b2c0a69:40573 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-07T10:43:51,747 INFO [RS:1;10641b2c0a69:41401 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-07T10:43:51,747 INFO [RS:0;10641b2c0a69:40573 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,747 INFO [RS:2;10641b2c0a69:39013 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,747 INFO [RS:1;10641b2c0a69:41401 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,749 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-07T10:43:51,749 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-07T10:43:51,752 INFO [RS:1;10641b2c0a69:41401 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-07T10:43:51,755 INFO [RS:2;10641b2c0a69:39013 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-07T10:43:51,755 INFO [RS:0;10641b2c0a69:40573 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-07T10:43:51,755 INFO [RS:1;10641b2c0a69:41401 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-07T10:43:51,756 INFO [RS:0;10641b2c0a69:40573 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,756 INFO [RS:1;10641b2c0a69:41401 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,756 INFO [RS:2;10641b2c0a69:39013 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,756 DEBUG [RS:2;10641b2c0a69:39013 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,756 DEBUG [RS:0;10641b2c0a69:40573 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,756 DEBUG [RS:1;10641b2c0a69:41401 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,757 DEBUG [RS:2;10641b2c0a69:39013 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,757 DEBUG [RS:0;10641b2c0a69:40573 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,757 DEBUG [RS:1;10641b2c0a69:41401 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,757 DEBUG [RS:2;10641b2c0a69:39013 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,757 DEBUG [RS:0;10641b2c0a69:40573 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,757 DEBUG [RS:1;10641b2c0a69:41401 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,757 DEBUG [RS:0;10641b2c0a69:40573 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,757 DEBUG [RS:2;10641b2c0a69:39013 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,757 DEBUG [RS:1;10641b2c0a69:41401 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,757 DEBUG [RS:2;10641b2c0a69:39013 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,757 DEBUG [RS:0;10641b2c0a69:40573 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,757 DEBUG [RS:2;10641b2c0a69:39013 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0, corePoolSize=2, maxPoolSize=2 2024-11-07T10:43:51,757 DEBUG [RS:0;10641b2c0a69:40573 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0, corePoolSize=2, maxPoolSize=2 2024-11-07T10:43:51,757 DEBUG [RS:1;10641b2c0a69:41401 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,757 DEBUG [RS:0;10641b2c0a69:40573 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,757 DEBUG [RS:2;10641b2c0a69:39013 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,757 DEBUG [RS:1;10641b2c0a69:41401 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0, corePoolSize=2, maxPoolSize=2 2024-11-07T10:43:51,757 DEBUG [RS:2;10641b2c0a69:39013 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,757 DEBUG [RS:0;10641b2c0a69:40573 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,758 DEBUG [RS:1;10641b2c0a69:41401 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,758 DEBUG [RS:0;10641b2c0a69:40573 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,758 DEBUG [RS:2;10641b2c0a69:39013 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,758 DEBUG [RS:1;10641b2c0a69:41401 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,758 DEBUG [RS:0;10641b2c0a69:40573 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,758 DEBUG [RS:2;10641b2c0a69:39013 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,758 DEBUG [RS:1;10641b2c0a69:41401 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,758 DEBUG [RS:0;10641b2c0a69:40573 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,758 DEBUG [RS:2;10641b2c0a69:39013 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,758 DEBUG [RS:1;10641b2c0a69:41401 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,758 DEBUG [RS:1;10641b2c0a69:41401 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,758 DEBUG [RS:0;10641b2c0a69:40573 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,758 DEBUG [RS:2;10641b2c0a69:39013 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,758 DEBUG [RS:1;10641b2c0a69:41401 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/10641b2c0a69:0, corePoolSize=1, maxPoolSize=1 2024-11-07T10:43:51,758 DEBUG [RS:0;10641b2c0a69:40573 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/10641b2c0a69:0, corePoolSize=3, maxPoolSize=3 2024-11-07T10:43:51,758 DEBUG [RS:2;10641b2c0a69:39013 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/10641b2c0a69:0, corePoolSize=3, maxPoolSize=3 2024-11-07T10:43:51,758 DEBUG [RS:1;10641b2c0a69:41401 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/10641b2c0a69:0, corePoolSize=3, maxPoolSize=3 2024-11-07T10:43:51,758 DEBUG [RS:2;10641b2c0a69:39013 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/10641b2c0a69:0, corePoolSize=3, maxPoolSize=3 2024-11-07T10:43:51,758 DEBUG [RS:0;10641b2c0a69:40573 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/10641b2c0a69:0, corePoolSize=3, maxPoolSize=3 2024-11-07T10:43:51,758 DEBUG [RS:1;10641b2c0a69:41401 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/10641b2c0a69:0, corePoolSize=3, maxPoolSize=3 2024-11-07T10:43:51,759 INFO [RS:1;10641b2c0a69:41401 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,760 INFO [RS:1;10641b2c0a69:41401 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,760 INFO [RS:2;10641b2c0a69:39013 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,760 INFO [RS:1;10641b2c0a69:41401 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,760 INFO [RS:0;10641b2c0a69:40573 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,760 INFO [RS:2;10641b2c0a69:39013 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,760 INFO [RS:1;10641b2c0a69:41401 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,760 INFO [RS:1;10641b2c0a69:41401 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,760 INFO [RS:2;10641b2c0a69:39013 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,760 INFO [RS:0;10641b2c0a69:40573 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,760 INFO [RS:1;10641b2c0a69:41401 {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,41401,1730976230578-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T10:43:51,760 INFO [RS:2;10641b2c0a69:39013 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,760 INFO [RS:0;10641b2c0a69:40573 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,760 INFO [RS:2;10641b2c0a69:39013 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,760 INFO [RS:0;10641b2c0a69:40573 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,760 INFO [RS:0;10641b2c0a69:40573 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,760 INFO [RS:2;10641b2c0a69:39013 {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,39013,1730976230620-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T10:43:51,760 INFO [RS:0;10641b2c0a69:40573 {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,40573,1730976230476-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T10:43:51,781 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-07T10:43:51,781 INFO [RS:1;10641b2c0a69:41401 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-07T10:43:51,783 INFO [RS:2;10641b2c0a69:39013 {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,39013,1730976230620-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,783 INFO [RS:1;10641b2c0a69:41401 {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,41401,1730976230578-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,784 INFO [RS:2;10641b2c0a69:39013 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,784 INFO [RS:1;10641b2c0a69:41401 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,784 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.Replication(171): 10641b2c0a69,39013,1730976230620 started 2024-11-07T10:43:51,784 INFO [RS:1;10641b2c0a69:41401 {}] regionserver.Replication(171): 10641b2c0a69,41401,1730976230578 started 2024-11-07T10:43:51,787 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-07T10:43:51,787 INFO [RS:0;10641b2c0a69:40573 {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,40573,1730976230476-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,787 INFO [RS:0;10641b2c0a69:40573 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,787 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.Replication(171): 10641b2c0a69,40573,1730976230476 started 2024-11-07T10:43:51,807 INFO [RS:2;10641b2c0a69:39013 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,807 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(1482): Serving as 10641b2c0a69,39013,1730976230620, RpcServer on 10641b2c0a69/172.17.0.2:39013, sessionid=0x10019d731180003 2024-11-07T10:43:51,808 DEBUG [RS:2;10641b2c0a69:39013 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-07T10:43:51,808 DEBUG [RS:2;10641b2c0a69:39013 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 10641b2c0a69,39013,1730976230620 2024-11-07T10:43:51,808 DEBUG [RS:2;10641b2c0a69:39013 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '10641b2c0a69,39013,1730976230620' 2024-11-07T10:43:51,808 DEBUG [RS:2;10641b2c0a69:39013 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-07T10:43:51,808 INFO [RS:1;10641b2c0a69:41401 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,809 INFO [RS:1;10641b2c0a69:41401 {}] regionserver.HRegionServer(1482): Serving as 10641b2c0a69,41401,1730976230578, RpcServer on 10641b2c0a69/172.17.0.2:41401, sessionid=0x10019d731180002 2024-11-07T10:43:51,809 DEBUG [RS:1;10641b2c0a69:41401 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-07T10:43:51,809 DEBUG [RS:1;10641b2c0a69:41401 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 10641b2c0a69,41401,1730976230578 2024-11-07T10:43:51,809 DEBUG [RS:1;10641b2c0a69:41401 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '10641b2c0a69,41401,1730976230578' 2024-11-07T10:43:51,809 DEBUG [RS:1;10641b2c0a69:41401 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-07T10:43:51,809 DEBUG [RS:2;10641b2c0a69:39013 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-07T10:43:51,810 INFO [RS:0;10641b2c0a69:40573 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:51,810 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(1482): Serving as 10641b2c0a69,40573,1730976230476, RpcServer on 10641b2c0a69/172.17.0.2:40573, sessionid=0x10019d731180001 2024-11-07T10:43:51,810 DEBUG [RS:1;10641b2c0a69:41401 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-07T10:43:51,810 DEBUG [RS:0;10641b2c0a69:40573 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-07T10:43:51,810 DEBUG [RS:0;10641b2c0a69:40573 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 10641b2c0a69,40573,1730976230476 2024-11-07T10:43:51,810 DEBUG [RS:0;10641b2c0a69:40573 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '10641b2c0a69,40573,1730976230476' 2024-11-07T10:43:51,810 DEBUG [RS:0;10641b2c0a69:40573 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-07T10:43:51,810 DEBUG [RS:2;10641b2c0a69:39013 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-07T10:43:51,810 DEBUG [RS:2;10641b2c0a69:39013 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-07T10:43:51,811 DEBUG [RS:1;10641b2c0a69:41401 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-07T10:43:51,811 DEBUG [RS:2;10641b2c0a69:39013 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 10641b2c0a69,39013,1730976230620 2024-11-07T10:43:51,811 DEBUG [RS:1;10641b2c0a69:41401 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-07T10:43:51,811 DEBUG [RS:2;10641b2c0a69:39013 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '10641b2c0a69,39013,1730976230620' 2024-11-07T10:43:51,811 DEBUG [RS:1;10641b2c0a69:41401 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 10641b2c0a69,41401,1730976230578 2024-11-07T10:43:51,811 DEBUG [RS:2;10641b2c0a69:39013 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-07T10:43:51,811 DEBUG [RS:1;10641b2c0a69:41401 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '10641b2c0a69,41401,1730976230578' 2024-11-07T10:43:51,811 DEBUG [RS:1;10641b2c0a69:41401 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-07T10:43:51,811 DEBUG [RS:0;10641b2c0a69:40573 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-07T10:43:51,812 DEBUG [RS:2;10641b2c0a69:39013 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-07T10:43:51,812 DEBUG [RS:1;10641b2c0a69:41401 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-07T10:43:51,812 DEBUG [RS:0;10641b2c0a69:40573 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-07T10:43:51,812 DEBUG [RS:0;10641b2c0a69:40573 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-07T10:43:51,812 DEBUG [RS:0;10641b2c0a69:40573 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 10641b2c0a69,40573,1730976230476 2024-11-07T10:43:51,812 DEBUG [RS:0;10641b2c0a69:40573 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '10641b2c0a69,40573,1730976230476' 2024-11-07T10:43:51,812 DEBUG [RS:0;10641b2c0a69:40573 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-07T10:43:51,812 DEBUG [RS:1;10641b2c0a69:41401 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-07T10:43:51,812 INFO [RS:1;10641b2c0a69:41401 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-07T10:43:51,813 INFO [RS:1;10641b2c0a69:41401 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-07T10:43:51,813 DEBUG [RS:2;10641b2c0a69:39013 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-07T10:43:51,813 INFO [RS:2;10641b2c0a69:39013 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-07T10:43:51,813 INFO [RS:2;10641b2c0a69:39013 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-07T10:43:51,813 DEBUG [RS:0;10641b2c0a69:40573 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-07T10:43:51,813 DEBUG [RS:0;10641b2c0a69:40573 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-07T10:43:51,813 INFO [RS:0;10641b2c0a69:40573 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-07T10:43:51,813 INFO [RS:0;10641b2c0a69:40573 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-07T10:43:51,872 WARN [10641b2c0a69:43605 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-07T10:43:51,918 INFO [RS:0;10641b2c0a69:40573 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T10:43:51,918 INFO [RS:1;10641b2c0a69:41401 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T10:43:51,918 INFO [RS:2;10641b2c0a69:39013 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-07T10:43:51,921 INFO [RS:1;10641b2c0a69:41401 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=10641b2c0a69%2C41401%2C1730976230578, suffix=, logDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578, archiveDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/oldWALs, maxLogs=32 2024-11-07T10:43:51,921 INFO [RS:0;10641b2c0a69:40573 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=10641b2c0a69%2C40573%2C1730976230476, suffix=, logDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,40573,1730976230476, archiveDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/oldWALs, maxLogs=32 2024-11-07T10:43:51,921 INFO [RS:2;10641b2c0a69:39013 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=10641b2c0a69%2C39013%2C1730976230620, suffix=, logDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,39013,1730976230620, archiveDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/oldWALs, maxLogs=32 2024-11-07T10:43:51,941 DEBUG [RS:0;10641b2c0a69:40573 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,40573,1730976230476/10641b2c0a69%2C40573%2C1730976230476.1730976231927, exclude list is [], retry=0 2024-11-07T10:43:51,941 DEBUG [RS:2;10641b2c0a69:39013 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,39013,1730976230620/10641b2c0a69%2C39013%2C1730976230620.1730976231927, exclude list is [], retry=0 2024-11-07T10:43:51,941 DEBUG [RS:1;10641b2c0a69:41401 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578/10641b2c0a69%2C41401%2C1730976230578.1730976231927, exclude list is [], retry=0 2024-11-07T10:43:51,946 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33357,DS-ca0d026c-750a-4349-8e6f-ce26c9f0979a,DISK] 2024-11-07T10:43:51,946 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41673,DS-4666ca76-3f5e-48c3-8d05-ac47a18a697b,DISK] 2024-11-07T10:43:51,946 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40045,DS-5d0e97fa-7109-4597-9d71-8fb432e0463c,DISK] 2024-11-07T10:43:51,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41673,DS-4666ca76-3f5e-48c3-8d05-ac47a18a697b,DISK] 2024-11-07T10:43:51,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33357,DS-ca0d026c-750a-4349-8e6f-ce26c9f0979a,DISK] 2024-11-07T10:43:51,948 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33357,DS-ca0d026c-750a-4349-8e6f-ce26c9f0979a,DISK] 2024-11-07T10:43:51,948 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40045,DS-5d0e97fa-7109-4597-9d71-8fb432e0463c,DISK] 2024-11-07T10:43:51,948 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40045,DS-5d0e97fa-7109-4597-9d71-8fb432e0463c,DISK] 2024-11-07T10:43:51,948 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41673,DS-4666ca76-3f5e-48c3-8d05-ac47a18a697b,DISK] 2024-11-07T10:43:51,962 INFO [RS:2;10641b2c0a69:39013 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,39013,1730976230620/10641b2c0a69%2C39013%2C1730976230620.1730976231927 2024-11-07T10:43:51,965 DEBUG [RS:2;10641b2c0a69:39013 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34291:34291),(127.0.0.1/127.0.0.1:41569:41569),(127.0.0.1/127.0.0.1:33799:33799)] 2024-11-07T10:43:51,966 INFO [RS:1;10641b2c0a69:41401 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578/10641b2c0a69%2C41401%2C1730976230578.1730976231927 2024-11-07T10:43:51,968 DEBUG [RS:1;10641b2c0a69:41401 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33799:33799),(127.0.0.1/127.0.0.1:41569:41569),(127.0.0.1/127.0.0.1:34291:34291)] 2024-11-07T10:43:51,969 INFO [RS:0;10641b2c0a69:40573 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,40573,1730976230476/10641b2c0a69%2C40573%2C1730976230476.1730976231927 2024-11-07T10:43:51,970 DEBUG [RS:0;10641b2c0a69:40573 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34291:34291),(127.0.0.1/127.0.0.1:41569:41569),(127.0.0.1/127.0.0.1:33799:33799)] 2024-11-07T10:43:52,124 DEBUG [10641b2c0a69:43605 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-07T10:43:52,132 DEBUG [10641b2c0a69:43605 {}] balancer.BalancerClusterState(204): Hosts are {10641b2c0a69=0} racks are {/default-rack=0} 2024-11-07T10:43:52,139 DEBUG [10641b2c0a69:43605 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-07T10:43:52,139 DEBUG [10641b2c0a69:43605 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-07T10:43:52,139 DEBUG [10641b2c0a69:43605 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-07T10:43:52,139 DEBUG [10641b2c0a69:43605 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-07T10:43:52,139 DEBUG [10641b2c0a69:43605 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-07T10:43:52,139 DEBUG [10641b2c0a69:43605 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-07T10:43:52,139 INFO [10641b2c0a69:43605 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-07T10:43:52,139 INFO [10641b2c0a69:43605 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-07T10:43:52,139 INFO [10641b2c0a69:43605 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-07T10:43:52,139 DEBUG [10641b2c0a69:43605 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-07T10:43:52,147 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=10641b2c0a69,41401,1730976230578 2024-11-07T10:43:52,154 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 10641b2c0a69,41401,1730976230578, state=OPENING 2024-11-07T10:43:52,158 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-07T10:43:52,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:43:52,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41401-0x10019d731180002, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:43:52,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:43:52,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:43:52,161 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:43:52,161 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:43:52,161 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:43:52,162 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:43:52,163 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T10:43:52,165 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=10641b2c0a69,41401,1730976230578}] 2024-11-07T10:43:52,340 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-07T10:43:52,342 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54169, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-07T10:43:52,354 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-07T10:43:52,354 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T10:43:52,354 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-07T10:43:52,357 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=10641b2c0a69%2C41401%2C1730976230578.meta, suffix=.meta, logDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578, archiveDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/oldWALs, maxLogs=32 2024-11-07T10:43:52,377 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578/10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, exclude list is [], retry=0 2024-11-07T10:43:52,381 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41673,DS-4666ca76-3f5e-48c3-8d05-ac47a18a697b,DISK] 2024-11-07T10:43:52,381 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40045,DS-5d0e97fa-7109-4597-9d71-8fb432e0463c,DISK] 2024-11-07T10:43:52,381 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33357,DS-ca0d026c-750a-4349-8e6f-ce26c9f0979a,DISK] 2024-11-07T10:43:52,385 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578/10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta 2024-11-07T10:43:52,387 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33799:33799),(127.0.0.1/127.0.0.1:41569:41569),(127.0.0.1/127.0.0.1:34291:34291)] 2024-11-07T10:43:52,387 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-07T10:43:52,389 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-07T10:43:52,391 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-07T10:43:52,397 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-07T10:43:52,401 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-07T10:43:52,401 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:52,402 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-07T10:43:52,402 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-07T10:43:52,405 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T10:43:52,406 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T10:43:52,406 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:52,407 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:43:52,408 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-07T10:43:52,409 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T10:43:52,409 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:52,410 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:43:52,410 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T10:43:52,412 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T10:43:52,412 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:52,413 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:43:52,413 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T10:43:52,415 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T10:43:52,415 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:52,415 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:43:52,416 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T10:43:52,417 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740 2024-11-07T10:43:52,419 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740 2024-11-07T10:43:52,422 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T10:43:52,422 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T10:43:52,423 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T10:43:52,425 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T10:43:52,427 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64810136, jitterRate=-0.034253716468811035}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T10:43:52,427 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-07T10:43:52,428 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1730976232402Writing region info on filesystem at 1730976232402Initializing all the Stores at 1730976232404 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730976232404Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730976232405 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976232405Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730976232405Cleaning up temporary data from old regions at 1730976232422 (+17 ms)Running coprocessor post-open hooks at 1730976232427 (+5 ms)Region opened successfully at 1730976232428 (+1 ms) 2024-11-07T10:43:52,435 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1730976232331 2024-11-07T10:43:52,446 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-07T10:43:52,447 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-07T10:43:52,448 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,41401,1730976230578 2024-11-07T10:43:52,450 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 10641b2c0a69,41401,1730976230578, state=OPEN 2024-11-07T10:43:52,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:43:52,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:43:52,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:43:52,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41401-0x10019d731180002, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:43:52,453 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:43:52,453 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:43:52,453 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:43:52,453 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:43:52,453 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=10641b2c0a69,41401,1730976230578 2024-11-07T10:43:52,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-07T10:43:52,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=10641b2c0a69,41401,1730976230578 in 289 msec 2024-11-07T10:43:52,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-07T10:43:52,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 761 msec 2024-11-07T10:43:52,467 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-07T10:43:52,467 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-07T10:43:52,489 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:43:52,491 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:43:52,521 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T10:43:52,523 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50685, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T10:43:52,544 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0930 sec 2024-11-07T10:43:52,545 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1730976232544, completionTime=-1 2024-11-07T10:43:52,547 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-07T10:43:52,547 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-07T10:43:52,577 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-07T10:43:52,577 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730976292577 2024-11-07T10:43:52,577 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730976352577 2024-11-07T10:43:52,577 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 29 msec 2024-11-07T10:43:52,579 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-07T10:43:52,588 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,43605,1730976229810-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:52,589 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,43605,1730976229810-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:52,589 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,43605,1730976229810-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:52,591 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-10641b2c0a69:43605, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:52,591 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:52,592 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:52,601 DEBUG [master/10641b2c0a69:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-07T10:43:52,626 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.931sec 2024-11-07T10:43:52,628 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-07T10:43:52,630 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-07T10:43:52,631 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-07T10:43:52,632 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-07T10:43:52,632 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-07T10:43:52,633 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,43605,1730976229810-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-07T10:43:52,633 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,43605,1730976229810-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-07T10:43:52,639 DEBUG [master/10641b2c0a69:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-07T10:43:52,640 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-07T10:43:52,640 INFO [master/10641b2c0a69:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=10641b2c0a69,43605,1730976229810-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-07T10:43:52,657 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17aba086, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T10:43:52,662 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-07T10:43:52,662 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-07T10:43:52,666 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 10641b2c0a69,43605,-1 for getting cluster id 2024-11-07T10:43:52,669 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-07T10:43:52,678 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '81cac59c-8db3-42bf-8d2a-6a03e22005b3' 2024-11-07T10:43:52,680 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-07T10:43:52,681 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "81cac59c-8db3-42bf-8d2a-6a03e22005b3" 2024-11-07T10:43:52,684 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e659bb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T10:43:52,684 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [10641b2c0a69,43605,-1] 2024-11-07T10:43:52,687 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-07T10:43:52,689 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T10:43:52,690 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35562, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T10:43:52,693 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63b8549a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-07T10:43:52,693 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:43:52,700 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:43:52,700 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T10:43:52,703 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47124, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T10:43:52,746 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=10641b2c0a69,43605,1730976229810 2024-11-07T10:43:52,748 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-07T10:43:52,752 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-07T10:43:52,757 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 10641b2c0a69,43605,1730976229810 2024-11-07T10:43:52,760 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7609c274 2024-11-07T10:43:52,764 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-07T10:43:52,767 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35574, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T10:43:52,773 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43605 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-07T10:43:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43605 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-07T10:43:52,784 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-07T10:43:52,786 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43605 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true" procId is: 4 2024-11-07T10:43:52,787 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:52,789 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-07T10:43:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T10:43:52,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741837_1013 (size=1020) 2024-11-07T10:43:52,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741837_1013 (size=1020) 2024-11-07T10:43:52,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741837_1013 (size=1020) 2024-11-07T10:43:52,807 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => dbe69e720cc4f2a4a4bc41171b6b3c32, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1730976232769.dbe69e720cc4f2a4a4bc41171b6b3c32.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 2024-11-07T10:43:52,808 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7a68cc5b6ea7d9e2214ab42d65b16eba, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 2024-11-07T10:43:52,808 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => bcc3491e6200809e4c9c7d529da2c4cf, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1730976232769.bcc3491e6200809e4c9c7d529da2c4cf.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 2024-11-07T10:43:52,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741838_1014 (size=107) 2024-11-07T10:43:52,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741838_1014 (size=107) 2024-11-07T10:43:52,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741838_1014 (size=107) 2024-11-07T10:43:52,827 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:52,827 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 7a68cc5b6ea7d9e2214ab42d65b16eba, disabling compactions & flushes 2024-11-07T10:43:52,828 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba. 2024-11-07T10:43:52,828 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba. 2024-11-07T10:43:52,828 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba. after waiting 0 ms 2024-11-07T10:43:52,828 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba. 2024-11-07T10:43:52,828 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba. 2024-11-07T10:43:52,828 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7a68cc5b6ea7d9e2214ab42d65b16eba: Waiting for close lock at 1730976232827Disabling compacts and flushes for region at 1730976232827Disabling writes for close at 1730976232828 (+1 ms)Writing region close event to WAL at 1730976232828Closed at 1730976232828 2024-11-07T10:43:52,829 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d4b381fbb1d4cd388f45774fca53b2eb, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 2024-11-07T10:43:52,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741840_1016 (size=110) 2024-11-07T10:43:52,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741839_1015 (size=110) 2024-11-07T10:43:52,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741839_1015 (size=110) 2024-11-07T10:43:52,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741839_1015 (size=110) 2024-11-07T10:43:52,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741840_1016 (size=110) 2024-11-07T10:43:52,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741840_1016 (size=110) 2024-11-07T10:43:52,838 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1730976232769.bcc3491e6200809e4c9c7d529da2c4cf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:52,838 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing bcc3491e6200809e4c9c7d529da2c4cf, disabling compactions & flushes 2024-11-07T10:43:52,838 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1730976232769.bcc3491e6200809e4c9c7d529da2c4cf. 2024-11-07T10:43:52,838 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1730976232769.bcc3491e6200809e4c9c7d529da2c4cf. 2024-11-07T10:43:52,838 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1730976232769.bcc3491e6200809e4c9c7d529da2c4cf. after waiting 0 ms 2024-11-07T10:43:52,838 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1730976232769.bcc3491e6200809e4c9c7d529da2c4cf. 2024-11-07T10:43:52,838 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1730976232769.bcc3491e6200809e4c9c7d529da2c4cf. 2024-11-07T10:43:52,838 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for bcc3491e6200809e4c9c7d529da2c4cf: Waiting for close lock at 1730976232838Disabling compacts and flushes for region at 1730976232838Disabling writes for close at 1730976232838Writing region close event to WAL at 1730976232838Closed at 1730976232838 2024-11-07T10:43:52,839 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 3f6021bbda5b8b7b00318a57f02cd77d, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 2024-11-07T10:43:52,840 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1730976232769.dbe69e720cc4f2a4a4bc41171b6b3c32.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:52,840 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing dbe69e720cc4f2a4a4bc41171b6b3c32, disabling compactions & flushes 2024-11-07T10:43:52,841 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1730976232769.dbe69e720cc4f2a4a4bc41171b6b3c32. 2024-11-07T10:43:52,841 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1730976232769.dbe69e720cc4f2a4a4bc41171b6b3c32. 2024-11-07T10:43:52,841 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1730976232769.dbe69e720cc4f2a4a4bc41171b6b3c32. after waiting 0 ms 2024-11-07T10:43:52,841 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1730976232769.dbe69e720cc4f2a4a4bc41171b6b3c32. 2024-11-07T10:43:52,841 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1730976232769.dbe69e720cc4f2a4a4bc41171b6b3c32. 2024-11-07T10:43:52,841 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for dbe69e720cc4f2a4a4bc41171b6b3c32: Waiting for close lock at 1730976232840Disabling compacts and flushes for region at 1730976232840Disabling writes for close at 1730976232841 (+1 ms)Writing region close event to WAL at 1730976232841Closed at 1730976232841 2024-11-07T10:43:52,842 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => ddeeab6f2e44d85eb1ee916d3c5e8e9f, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1730976232769.ddeeab6f2e44d85eb1ee916d3c5e8e9f.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 2024-11-07T10:43:52,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741841_1017 (size=110) 2024-11-07T10:43:52,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741841_1017 (size=110) 2024-11-07T10:43:52,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741841_1017 (size=110) 2024-11-07T10:43:52,847 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:52,847 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing d4b381fbb1d4cd388f45774fca53b2eb, disabling compactions & flushes 2024-11-07T10:43:52,848 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb. 2024-11-07T10:43:52,848 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb. 2024-11-07T10:43:52,848 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb. after waiting 0 ms 2024-11-07T10:43:52,848 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb. 2024-11-07T10:43:52,848 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb. 2024-11-07T10:43:52,848 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for d4b381fbb1d4cd388f45774fca53b2eb: Waiting for close lock at 1730976232847Disabling compacts and flushes for region at 1730976232847Disabling writes for close at 1730976232848 (+1 ms)Writing region close event to WAL at 1730976232848Closed at 1730976232848 2024-11-07T10:43:52,848 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => db89a463448eb182960960c83f4e0b16, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 2024-11-07T10:43:52,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741842_1018 (size=110) 2024-11-07T10:43:52,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741843_1019 (size=110) 2024-11-07T10:43:52,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741843_1019 (size=110) 2024-11-07T10:43:52,859 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1730976232769.ddeeab6f2e44d85eb1ee916d3c5e8e9f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:52,859 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing ddeeab6f2e44d85eb1ee916d3c5e8e9f, disabling compactions & flushes 2024-11-07T10:43:52,859 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1730976232769.ddeeab6f2e44d85eb1ee916d3c5e8e9f. 2024-11-07T10:43:52,859 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1730976232769.ddeeab6f2e44d85eb1ee916d3c5e8e9f. 2024-11-07T10:43:52,859 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1730976232769.ddeeab6f2e44d85eb1ee916d3c5e8e9f. after waiting 0 ms 2024-11-07T10:43:52,859 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1730976232769.ddeeab6f2e44d85eb1ee916d3c5e8e9f. 2024-11-07T10:43:52,859 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1730976232769.ddeeab6f2e44d85eb1ee916d3c5e8e9f. 2024-11-07T10:43:52,859 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for ddeeab6f2e44d85eb1ee916d3c5e8e9f: Waiting for close lock at 1730976232859Disabling compacts and flushes for region at 1730976232859Disabling writes for close at 1730976232859Writing region close event to WAL at 1730976232859Closed at 1730976232859 2024-11-07T10:43:52,860 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => c3ab41a5354d389521b694050824788e, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1730976232769.c3ab41a5354d389521b694050824788e.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 2024-11-07T10:43:52,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741842_1018 (size=110) 2024-11-07T10:43:52,862 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:52,862 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 3f6021bbda5b8b7b00318a57f02cd77d, disabling compactions & flushes 2024-11-07T10:43:52,862 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d. 2024-11-07T10:43:52,862 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d. 2024-11-07T10:43:52,862 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d. after waiting 0 ms 2024-11-07T10:43:52,862 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d. 2024-11-07T10:43:52,862 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d. 2024-11-07T10:43:52,862 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 3f6021bbda5b8b7b00318a57f02cd77d: Waiting for close lock at 1730976232862Disabling compacts and flushes for region at 1730976232862Disabling writes for close at 1730976232862Writing region close event to WAL at 1730976232862Closed at 1730976232862 2024-11-07T10:43:52,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741844_1020 (size=110) 2024-11-07T10:43:52,863 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => f89575b73864cac8b7b164f417ef378f, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1730976232769.f89575b73864cac8b7b164f417ef378f.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 2024-11-07T10:43:52,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741844_1020 (size=110) 2024-11-07T10:43:52,865 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:52,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741842_1018 (size=110) 2024-11-07T10:43:52,865 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing db89a463448eb182960960c83f4e0b16, disabling compactions & flushes 2024-11-07T10:43:52,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741843_1019 (size=110) 2024-11-07T10:43:52,865 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16. 2024-11-07T10:43:52,865 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16. 2024-11-07T10:43:52,865 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16. after waiting 0 ms 2024-11-07T10:43:52,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741844_1020 (size=110) 2024-11-07T10:43:52,865 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16. 2024-11-07T10:43:52,866 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16. 2024-11-07T10:43:52,866 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for db89a463448eb182960960c83f4e0b16: Waiting for close lock at 1730976232865Disabling compacts and flushes for region at 1730976232865Disabling writes for close at 1730976232865Writing region close event to WAL at 1730976232866 (+1 ms)Closed at 1730976232866 2024-11-07T10:43:52,866 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 72bf61b738a2f9b35913a192fc32815b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 2024-11-07T10:43:52,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741845_1021 (size=110) 2024-11-07T10:43:52,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741845_1021 (size=110) 2024-11-07T10:43:52,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741845_1021 (size=110) 2024-11-07T10:43:52,876 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1730976232769.c3ab41a5354d389521b694050824788e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:52,876 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing c3ab41a5354d389521b694050824788e, disabling compactions & flushes 2024-11-07T10:43:52,876 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1730976232769.c3ab41a5354d389521b694050824788e. 2024-11-07T10:43:52,876 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1730976232769.c3ab41a5354d389521b694050824788e. 2024-11-07T10:43:52,876 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1730976232769.c3ab41a5354d389521b694050824788e. after waiting 0 ms 2024-11-07T10:43:52,876 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1730976232769.c3ab41a5354d389521b694050824788e. 2024-11-07T10:43:52,876 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1730976232769.c3ab41a5354d389521b694050824788e. 2024-11-07T10:43:52,877 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for c3ab41a5354d389521b694050824788e: Waiting for close lock at 1730976232876Disabling compacts and flushes for region at 1730976232876Disabling writes for close at 1730976232876Writing region close event to WAL at 1730976232876Closed at 1730976232876 2024-11-07T10:43:52,877 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 0a7d18696a90cfed5af1fbdb2a706b65, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1730976232769.0a7d18696a90cfed5af1fbdb2a706b65.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 2024-11-07T10:43:52,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741846_1022 (size=110) 2024-11-07T10:43:52,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741846_1022 (size=110) 2024-11-07T10:43:52,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741846_1022 (size=110) 2024-11-07T10:43:52,884 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1730976232769.f89575b73864cac8b7b164f417ef378f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:52,884 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing f89575b73864cac8b7b164f417ef378f, disabling compactions & flushes 2024-11-07T10:43:52,884 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1730976232769.f89575b73864cac8b7b164f417ef378f. 2024-11-07T10:43:52,884 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1730976232769.f89575b73864cac8b7b164f417ef378f. 2024-11-07T10:43:52,884 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1730976232769.f89575b73864cac8b7b164f417ef378f. after waiting 0 ms 2024-11-07T10:43:52,884 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1730976232769.f89575b73864cac8b7b164f417ef378f. 2024-11-07T10:43:52,884 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1730976232769.f89575b73864cac8b7b164f417ef378f. 2024-11-07T10:43:52,884 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for f89575b73864cac8b7b164f417ef378f: Waiting for close lock at 1730976232884Disabling compacts and flushes for region at 1730976232884Disabling writes for close at 1730976232884Writing region close event to WAL at 1730976232884Closed at 1730976232884 2024-11-07T10:43:52,885 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1baabb362104b46ecb78fe236391ca6f, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1730976232769.1baabb362104b46ecb78fe236391ca6f.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 2024-11-07T10:43:52,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741847_1023 (size=110) 2024-11-07T10:43:52,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741847_1023 (size=110) 2024-11-07T10:43:52,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741847_1023 (size=110) 2024-11-07T10:43:52,898 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:52,898 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 72bf61b738a2f9b35913a192fc32815b, disabling compactions & flushes 2024-11-07T10:43:52,898 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b. 2024-11-07T10:43:52,898 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b. 2024-11-07T10:43:52,898 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b. after waiting 0 ms 2024-11-07T10:43:52,898 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b. 2024-11-07T10:43:52,898 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b. 2024-11-07T10:43:52,898 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 72bf61b738a2f9b35913a192fc32815b: Waiting for close lock at 1730976232898Disabling compacts and flushes for region at 1730976232898Disabling writes for close at 1730976232898Writing region close event to WAL at 1730976232898Closed at 1730976232898 2024-11-07T10:43:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T10:43:52,899 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8ecb70a5eda60ab5744089e2ffe59d71, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 2024-11-07T10:43:52,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741848_1024 (size=110) 2024-11-07T10:43:52,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741848_1024 (size=110) 2024-11-07T10:43:52,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741848_1024 (size=110) 2024-11-07T10:43:52,908 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1730976232769.0a7d18696a90cfed5af1fbdb2a706b65.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:52,908 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 0a7d18696a90cfed5af1fbdb2a706b65, disabling compactions & flushes 2024-11-07T10:43:52,908 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1730976232769.0a7d18696a90cfed5af1fbdb2a706b65. 2024-11-07T10:43:52,908 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1730976232769.0a7d18696a90cfed5af1fbdb2a706b65. 2024-11-07T10:43:52,908 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1730976232769.0a7d18696a90cfed5af1fbdb2a706b65. after waiting 0 ms 2024-11-07T10:43:52,909 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1730976232769.0a7d18696a90cfed5af1fbdb2a706b65. 2024-11-07T10:43:52,909 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1730976232769.0a7d18696a90cfed5af1fbdb2a706b65. 2024-11-07T10:43:52,909 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 0a7d18696a90cfed5af1fbdb2a706b65: Waiting for close lock at 1730976232908Disabling compacts and flushes for region at 1730976232908Disabling writes for close at 1730976232908Writing region close event to WAL at 1730976232909 (+1 ms)Closed at 1730976232909 2024-11-07T10:43:52,910 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 9b830813c6b6c3ebcdb59061af10faad, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1730976232769.9b830813c6b6c3ebcdb59061af10faad.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 2024-11-07T10:43:52,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741849_1025 (size=110) 2024-11-07T10:43:52,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741849_1025 (size=110) 2024-11-07T10:43:52,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741849_1025 (size=110) 2024-11-07T10:43:52,913 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1730976232769.1baabb362104b46ecb78fe236391ca6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:52,913 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 1baabb362104b46ecb78fe236391ca6f, disabling compactions & flushes 2024-11-07T10:43:52,913 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1730976232769.1baabb362104b46ecb78fe236391ca6f. 2024-11-07T10:43:52,913 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1730976232769.1baabb362104b46ecb78fe236391ca6f. 2024-11-07T10:43:52,913 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1730976232769.1baabb362104b46ecb78fe236391ca6f. after waiting 0 ms 2024-11-07T10:43:52,913 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1730976232769.1baabb362104b46ecb78fe236391ca6f. 2024-11-07T10:43:52,913 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1730976232769.1baabb362104b46ecb78fe236391ca6f. 2024-11-07T10:43:52,913 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 1baabb362104b46ecb78fe236391ca6f: Waiting for close lock at 1730976232913Disabling compacts and flushes for region at 1730976232913Disabling writes for close at 1730976232913Writing region close event to WAL at 1730976232913Closed at 1730976232913 2024-11-07T10:43:52,914 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => dec39ef6d94f3a519151705fce17adc4, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1730976232769.dec39ef6d94f3a519151705fce17adc4.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 2024-11-07T10:43:52,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741850_1026 (size=110) 2024-11-07T10:43:52,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741850_1026 (size=110) 2024-11-07T10:43:52,928 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:52,928 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 8ecb70a5eda60ab5744089e2ffe59d71, disabling compactions & flushes 2024-11-07T10:43:52,928 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71. 2024-11-07T10:43:52,928 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71. 2024-11-07T10:43:52,928 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71. after waiting 0 ms 2024-11-07T10:43:52,928 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71. 2024-11-07T10:43:52,928 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71. 2024-11-07T10:43:52,928 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8ecb70a5eda60ab5744089e2ffe59d71: Waiting for close lock at 1730976232928Disabling compacts and flushes for region at 1730976232928Disabling writes for close at 1730976232928Writing region close event to WAL at 1730976232928Closed at 1730976232928 2024-11-07T10:43:52,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741850_1026 (size=110) 2024-11-07T10:43:52,929 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 541874b6b96014b10121ede508f1f61f, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 2024-11-07T10:43:52,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741852_1028 (size=110) 2024-11-07T10:43:52,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741851_1027 (size=110) 2024-11-07T10:43:52,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741852_1028 (size=110) 2024-11-07T10:43:52,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741852_1028 (size=110) 2024-11-07T10:43:52,935 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1730976232769.9b830813c6b6c3ebcdb59061af10faad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:52,935 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 9b830813c6b6c3ebcdb59061af10faad, disabling compactions & flushes 2024-11-07T10:43:52,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741851_1027 (size=110) 2024-11-07T10:43:52,935 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1730976232769.9b830813c6b6c3ebcdb59061af10faad. 2024-11-07T10:43:52,935 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1730976232769.9b830813c6b6c3ebcdb59061af10faad. 2024-11-07T10:43:52,935 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1730976232769.9b830813c6b6c3ebcdb59061af10faad. after waiting 0 ms 2024-11-07T10:43:52,935 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1730976232769.9b830813c6b6c3ebcdb59061af10faad. 2024-11-07T10:43:52,936 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1730976232769.9b830813c6b6c3ebcdb59061af10faad. 2024-11-07T10:43:52,936 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 9b830813c6b6c3ebcdb59061af10faad: Waiting for close lock at 1730976232935Disabling compacts and flushes for region at 1730976232935Disabling writes for close at 1730976232935Writing region close event to WAL at 1730976232936 (+1 ms)Closed at 1730976232936 2024-11-07T10:43:52,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741851_1027 (size=110) 2024-11-07T10:43:52,936 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 64da674e3a9ec8e7458eb07874d9f204, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1730976232769.64da674e3a9ec8e7458eb07874d9f204.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 2024-11-07T10:43:52,937 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1730976232769.dec39ef6d94f3a519151705fce17adc4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:52,937 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing dec39ef6d94f3a519151705fce17adc4, disabling compactions & flushes 2024-11-07T10:43:52,937 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1730976232769.dec39ef6d94f3a519151705fce17adc4. 2024-11-07T10:43:52,937 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1730976232769.dec39ef6d94f3a519151705fce17adc4. 2024-11-07T10:43:52,937 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1730976232769.dec39ef6d94f3a519151705fce17adc4. after waiting 0 ms 2024-11-07T10:43:52,937 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1730976232769.dec39ef6d94f3a519151705fce17adc4. 2024-11-07T10:43:52,937 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1730976232769.dec39ef6d94f3a519151705fce17adc4. 2024-11-07T10:43:52,937 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for dec39ef6d94f3a519151705fce17adc4: Waiting for close lock at 1730976232937Disabling compacts and flushes for region at 1730976232937Disabling writes for close at 1730976232937Writing region close event to WAL at 1730976232937Closed at 1730976232937 2024-11-07T10:43:52,938 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 9fba8e9ebafee4e67bfa1295ac44d4a5, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1730976232769.9fba8e9ebafee4e67bfa1295ac44d4a5.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 2024-11-07T10:43:52,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741853_1029 (size=110) 2024-11-07T10:43:52,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741853_1029 (size=110) 2024-11-07T10:43:52,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741853_1029 (size=110) 2024-11-07T10:43:52,949 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:52,949 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 541874b6b96014b10121ede508f1f61f, disabling compactions & flushes 2024-11-07T10:43:52,949 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f. 2024-11-07T10:43:52,949 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f. 2024-11-07T10:43:52,949 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f. after waiting 0 ms 2024-11-07T10:43:52,949 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f. 2024-11-07T10:43:52,949 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f. 2024-11-07T10:43:52,949 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 541874b6b96014b10121ede508f1f61f: Waiting for close lock at 1730976232949Disabling compacts and flushes for region at 1730976232949Disabling writes for close at 1730976232949Writing region close event to WAL at 1730976232949Closed at 1730976232949 2024-11-07T10:43:52,950 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1701fae2bf725fdd94c16d1c6b8bc75f, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 2024-11-07T10:43:52,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741855_1031 (size=110) 2024-11-07T10:43:52,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741854_1030 (size=110) 2024-11-07T10:43:52,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741855_1031 (size=110) 2024-11-07T10:43:52,961 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1730976232769.9fba8e9ebafee4e67bfa1295ac44d4a5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:52,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741855_1031 (size=110) 2024-11-07T10:43:52,962 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 9fba8e9ebafee4e67bfa1295ac44d4a5, disabling compactions & flushes 2024-11-07T10:43:52,962 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1730976232769.9fba8e9ebafee4e67bfa1295ac44d4a5. 2024-11-07T10:43:52,962 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1730976232769.9fba8e9ebafee4e67bfa1295ac44d4a5. 2024-11-07T10:43:52,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741854_1030 (size=110) 2024-11-07T10:43:52,962 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1730976232769.9fba8e9ebafee4e67bfa1295ac44d4a5. after waiting 0 ms 2024-11-07T10:43:52,962 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1730976232769.9fba8e9ebafee4e67bfa1295ac44d4a5. 2024-11-07T10:43:52,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741854_1030 (size=110) 2024-11-07T10:43:52,962 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1730976232769.9fba8e9ebafee4e67bfa1295ac44d4a5. 2024-11-07T10:43:52,962 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 9fba8e9ebafee4e67bfa1295ac44d4a5: Waiting for close lock at 1730976232961Disabling compacts and flushes for region at 1730976232961Disabling writes for close at 1730976232962 (+1 ms)Writing region close event to WAL at 1730976232962Closed at 1730976232962 2024-11-07T10:43:52,963 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 2cf36f48b84f67b77887428ec6ad5b31, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1730976232769.2cf36f48b84f67b77887428ec6ad5b31.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 2024-11-07T10:43:52,963 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1730976232769.64da674e3a9ec8e7458eb07874d9f204.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:52,964 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 64da674e3a9ec8e7458eb07874d9f204, disabling compactions & flushes 2024-11-07T10:43:52,964 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1730976232769.64da674e3a9ec8e7458eb07874d9f204. 2024-11-07T10:43:52,964 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1730976232769.64da674e3a9ec8e7458eb07874d9f204. 2024-11-07T10:43:52,964 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1730976232769.64da674e3a9ec8e7458eb07874d9f204. after waiting 0 ms 2024-11-07T10:43:52,964 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1730976232769.64da674e3a9ec8e7458eb07874d9f204. 2024-11-07T10:43:52,964 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1730976232769.64da674e3a9ec8e7458eb07874d9f204. 2024-11-07T10:43:52,964 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 64da674e3a9ec8e7458eb07874d9f204: Waiting for close lock at 1730976232964Disabling compacts and flushes for region at 1730976232964Disabling writes for close at 1730976232964Writing region close event to WAL at 1730976232964Closed at 1730976232964 2024-11-07T10:43:52,965 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 69160268ea309aa55ece0cd8528b941d, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1730976232769.69160268ea309aa55ece0cd8528b941d.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 2024-11-07T10:43:52,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741856_1032 (size=110) 2024-11-07T10:43:52,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741856_1032 (size=110) 2024-11-07T10:43:52,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741856_1032 (size=110) 2024-11-07T10:43:52,971 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:52,971 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 1701fae2bf725fdd94c16d1c6b8bc75f, disabling compactions & flushes 2024-11-07T10:43:52,971 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f. 2024-11-07T10:43:52,971 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f. 2024-11-07T10:43:52,971 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f. after waiting 0 ms 2024-11-07T10:43:52,971 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f. 2024-11-07T10:43:52,971 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f. 2024-11-07T10:43:52,971 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1701fae2bf725fdd94c16d1c6b8bc75f: Waiting for close lock at 1730976232971Disabling compacts and flushes for region at 1730976232971Disabling writes for close at 1730976232971Writing region close event to WAL at 1730976232971Closed at 1730976232971 2024-11-07T10:43:52,972 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 257c2c76c61c100547acb4a778fb1fef, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 2024-11-07T10:43:52,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741857_1033 (size=110) 2024-11-07T10:43:52,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741857_1033 (size=110) 2024-11-07T10:43:52,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741857_1033 (size=110) 2024-11-07T10:43:52,977 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1730976232769.2cf36f48b84f67b77887428ec6ad5b31.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:52,977 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 2cf36f48b84f67b77887428ec6ad5b31, disabling compactions & flushes 2024-11-07T10:43:52,977 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1730976232769.2cf36f48b84f67b77887428ec6ad5b31. 2024-11-07T10:43:52,977 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1730976232769.2cf36f48b84f67b77887428ec6ad5b31. 2024-11-07T10:43:52,977 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1730976232769.2cf36f48b84f67b77887428ec6ad5b31. after waiting 0 ms 2024-11-07T10:43:52,977 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1730976232769.2cf36f48b84f67b77887428ec6ad5b31. 2024-11-07T10:43:52,977 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1730976232769.2cf36f48b84f67b77887428ec6ad5b31. 2024-11-07T10:43:52,977 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 2cf36f48b84f67b77887428ec6ad5b31: Waiting for close lock at 1730976232977Disabling compacts and flushes for region at 1730976232977Disabling writes for close at 1730976232977Writing region close event to WAL at 1730976232977Closed at 1730976232977 2024-11-07T10:43:52,978 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1dd5c615432f6d3976b7d3e591d0a96a, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1730976232769.1dd5c615432f6d3976b7d3e591d0a96a.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 2024-11-07T10:43:52,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741858_1034 (size=110) 2024-11-07T10:43:52,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741858_1034 (size=110) 2024-11-07T10:43:52,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741858_1034 (size=110) 2024-11-07T10:43:52,989 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1730976232769.69160268ea309aa55ece0cd8528b941d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:52,989 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 69160268ea309aa55ece0cd8528b941d, disabling compactions & flushes 2024-11-07T10:43:52,989 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1730976232769.69160268ea309aa55ece0cd8528b941d. 2024-11-07T10:43:52,989 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1730976232769.69160268ea309aa55ece0cd8528b941d. 2024-11-07T10:43:52,989 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1730976232769.69160268ea309aa55ece0cd8528b941d. after waiting 0 ms 2024-11-07T10:43:52,989 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1730976232769.69160268ea309aa55ece0cd8528b941d. 2024-11-07T10:43:52,989 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1730976232769.69160268ea309aa55ece0cd8528b941d. 2024-11-07T10:43:52,989 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 69160268ea309aa55ece0cd8528b941d: Waiting for close lock at 1730976232989Disabling compacts and flushes for region at 1730976232989Disabling writes for close at 1730976232989Writing region close event to WAL at 1730976232989Closed at 1730976232989 2024-11-07T10:43:52,990 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 0b29f5d988381aa77506fa3b92944854, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1730976232769.0b29f5d988381aa77506fa3b92944854.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 2024-11-07T10:43:52,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741859_1035 (size=110) 2024-11-07T10:43:52,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741859_1035 (size=110) 2024-11-07T10:43:52,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741859_1035 (size=110) 2024-11-07T10:43:52,992 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:52,992 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 257c2c76c61c100547acb4a778fb1fef, disabling compactions & flushes 2024-11-07T10:43:52,992 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef. 2024-11-07T10:43:52,992 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef. 2024-11-07T10:43:52,992 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef. after waiting 0 ms 2024-11-07T10:43:52,993 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef. 2024-11-07T10:43:52,993 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef. 2024-11-07T10:43:52,993 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 257c2c76c61c100547acb4a778fb1fef: Waiting for close lock at 1730976232992Disabling compacts and flushes for region at 1730976232992Disabling writes for close at 1730976232993 (+1 ms)Writing region close event to WAL at 1730976232993Closed at 1730976232993 2024-11-07T10:43:52,994 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8f449585c6b7b82f4442604100a49644, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 2024-11-07T10:43:52,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741860_1036 (size=110) 2024-11-07T10:43:52,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741860_1036 (size=110) 2024-11-07T10:43:52,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741860_1036 (size=110) 2024-11-07T10:43:52,999 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1730976232769.1dd5c615432f6d3976b7d3e591d0a96a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:52,999 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 1dd5c615432f6d3976b7d3e591d0a96a, disabling compactions & flushes 2024-11-07T10:43:52,999 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1730976232769.1dd5c615432f6d3976b7d3e591d0a96a. 2024-11-07T10:43:52,999 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1730976232769.1dd5c615432f6d3976b7d3e591d0a96a. 2024-11-07T10:43:52,999 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1730976232769.1dd5c615432f6d3976b7d3e591d0a96a. after waiting 0 ms 2024-11-07T10:43:52,999 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1730976232769.1dd5c615432f6d3976b7d3e591d0a96a. 2024-11-07T10:43:52,999 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1730976232769.1dd5c615432f6d3976b7d3e591d0a96a. 2024-11-07T10:43:52,999 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 1dd5c615432f6d3976b7d3e591d0a96a: Waiting for close lock at 1730976232999Disabling compacts and flushes for region at 1730976232999Disabling writes for close at 1730976232999Writing region close event to WAL at 1730976232999Closed at 1730976232999 2024-11-07T10:43:53,000 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => c71057666e459779c2fa5a4311f4e99c, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1730976232769.c71057666e459779c2fa5a4311f4e99c.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393 2024-11-07T10:43:53,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741861_1037 (size=110) 2024-11-07T10:43:53,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741861_1037 (size=110) 2024-11-07T10:43:53,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741861_1037 (size=110) 2024-11-07T10:43:53,006 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1730976232769.0b29f5d988381aa77506fa3b92944854.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:53,006 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 0b29f5d988381aa77506fa3b92944854, disabling compactions & flushes 2024-11-07T10:43:53,006 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1730976232769.0b29f5d988381aa77506fa3b92944854. 2024-11-07T10:43:53,006 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1730976232769.0b29f5d988381aa77506fa3b92944854. 2024-11-07T10:43:53,006 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1730976232769.0b29f5d988381aa77506fa3b92944854. after waiting 0 ms 2024-11-07T10:43:53,006 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1730976232769.0b29f5d988381aa77506fa3b92944854. 2024-11-07T10:43:53,006 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1730976232769.0b29f5d988381aa77506fa3b92944854. 2024-11-07T10:43:53,006 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 0b29f5d988381aa77506fa3b92944854: Waiting for close lock at 1730976233006Disabling compacts and flushes for region at 1730976233006Disabling writes for close at 1730976233006Writing region close event to WAL at 1730976233006Closed at 1730976233006 2024-11-07T10:43:53,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741862_1038 (size=110) 2024-11-07T10:43:53,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741862_1038 (size=110) 2024-11-07T10:43:53,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741862_1038 (size=110) 2024-11-07T10:43:53,015 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:53,015 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 8f449585c6b7b82f4442604100a49644, disabling compactions & flushes 2024-11-07T10:43:53,015 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644. 2024-11-07T10:43:53,015 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644. 2024-11-07T10:43:53,015 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644. after waiting 0 ms 2024-11-07T10:43:53,015 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644. 2024-11-07T10:43:53,015 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644. 2024-11-07T10:43:53,015 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8f449585c6b7b82f4442604100a49644: Waiting for close lock at 1730976233015Disabling compacts and flushes for region at 1730976233015Disabling writes for close at 1730976233015Writing region close event to WAL at 1730976233015Closed at 1730976233015 2024-11-07T10:43:53,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741863_1039 (size=107) 2024-11-07T10:43:53,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741863_1039 (size=107) 2024-11-07T10:43:53,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741863_1039 (size=107) 2024-11-07T10:43:53,020 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1730976232769.c71057666e459779c2fa5a4311f4e99c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:53,020 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing c71057666e459779c2fa5a4311f4e99c, disabling compactions & flushes 2024-11-07T10:43:53,020 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1730976232769.c71057666e459779c2fa5a4311f4e99c. 2024-11-07T10:43:53,020 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1730976232769.c71057666e459779c2fa5a4311f4e99c. 2024-11-07T10:43:53,020 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1730976232769.c71057666e459779c2fa5a4311f4e99c. after waiting 0 ms 2024-11-07T10:43:53,020 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1730976232769.c71057666e459779c2fa5a4311f4e99c. 2024-11-07T10:43:53,020 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1730976232769.c71057666e459779c2fa5a4311f4e99c. 2024-11-07T10:43:53,020 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for c71057666e459779c2fa5a4311f4e99c: Waiting for close lock at 1730976233020Disabling compacts and flushes for region at 1730976233020Disabling writes for close at 1730976233020Writing region close event to WAL at 1730976233020Closed at 1730976233020 2024-11-07T10:43:53,023 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-07T10:43:53,029 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba.","families":{"info":[{"qualifier":"regioninfo","vlen":106,"tag":[],"timestamp":"1730976233023"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730976233023"}]},"ts":"1730976233023"} 2024-11-07T10:43:53,029 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1730976232769.bcc3491e6200809e4c9c7d529da2c4cf.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1730976233023"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730976233023"}]},"ts":"1730976233023"} 2024-11-07T10:43:53,029 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1730976232769.dbe69e720cc4f2a4a4bc41171b6b3c32.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1730976233023"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730976233023"}]},"ts":"1730976233023"} 2024-11-07T10:43:53,030 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1730976233023"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730976233023"}]},"ts":"1730976233023"} 2024-11-07T10:43:53,030 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1730976232769.ddeeab6f2e44d85eb1ee916d3c5e8e9f.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1730976233023"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730976233023"}]},"ts":"1730976233023"} 2024-11-07T10:43:53,030 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1730976233023"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730976233023"}]},"ts":"1730976233023"} 2024-11-07T10:43:53,030 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1730976233023"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730976233023"}]},"ts":"1730976233023"} 2024-11-07T10:43:53,030 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1730976232769.c3ab41a5354d389521b694050824788e.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1730976233023"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730976233023"}]},"ts":"1730976233023"} 2024-11-07T10:43:53,031 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1730976232769.f89575b73864cac8b7b164f417ef378f.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1730976233023"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730976233023"}]},"ts":"1730976233023"} 2024-11-07T10:43:53,031 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1730976233023"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730976233023"}]},"ts":"1730976233023"} 2024-11-07T10:43:53,031 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1730976232769.0a7d18696a90cfed5af1fbdb2a706b65.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1730976233023"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730976233023"}]},"ts":"1730976233023"} 2024-11-07T10:43:53,031 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1730976232769.1baabb362104b46ecb78fe236391ca6f.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1730976233023"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730976233023"}]},"ts":"1730976233023"} 2024-11-07T10:43:53,031 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1730976233023"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730976233023"}]},"ts":"1730976233023"} 2024-11-07T10:43:53,032 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1730976232769.9b830813c6b6c3ebcdb59061af10faad.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1730976233023"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730976233023"}]},"ts":"1730976233023"} 2024-11-07T10:43:53,032 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1730976232769.dec39ef6d94f3a519151705fce17adc4.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1730976233023"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730976233023"}]},"ts":"1730976233023"} 2024-11-07T10:43:53,032 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1730976233023"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730976233023"}]},"ts":"1730976233023"} 2024-11-07T10:43:53,032 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1730976232769.9fba8e9ebafee4e67bfa1295ac44d4a5.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1730976233023"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730976233023"}]},"ts":"1730976233023"} 2024-11-07T10:43:53,032 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1730976232769.64da674e3a9ec8e7458eb07874d9f204.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1730976233023"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730976233023"}]},"ts":"1730976233023"} 2024-11-07T10:43:53,033 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1730976233023"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730976233023"}]},"ts":"1730976233023"} 2024-11-07T10:43:53,033 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1730976232769.2cf36f48b84f67b77887428ec6ad5b31.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1730976233023"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730976233023"}]},"ts":"1730976233023"} 2024-11-07T10:43:53,033 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1730976232769.69160268ea309aa55ece0cd8528b941d.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1730976233023"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730976233023"}]},"ts":"1730976233023"} 2024-11-07T10:43:53,033 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1730976233023"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730976233023"}]},"ts":"1730976233023"} 2024-11-07T10:43:53,033 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1730976232769.1dd5c615432f6d3976b7d3e591d0a96a.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1730976233023"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730976233023"}]},"ts":"1730976233023"} 2024-11-07T10:43:53,034 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1730976232769.0b29f5d988381aa77506fa3b92944854.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1730976233023"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730976233023"}]},"ts":"1730976233023"} 2024-11-07T10:43:53,034 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1730976233023"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730976233023"}]},"ts":"1730976233023"} 2024-11-07T10:43:53,034 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1730976232769.c71057666e459779c2fa5a4311f4e99c.","families":{"info":[{"qualifier":"regioninfo","vlen":106,"tag":[],"timestamp":"1730976233023"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1730976233023"}]},"ts":"1730976233023"} 2024-11-07T10:43:53,079 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-07T10:43:53,081 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-07T10:43:53,084 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730976233081"}]},"ts":"1730976233081"} 2024-11-07T10:43:53,089 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, state=ENABLING in hbase:meta 2024-11-07T10:43:53,090 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {10641b2c0a69=0} racks are {/default-rack=0} 2024-11-07T10:43:53,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T10:43:53,134 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-07T10:43:53,134 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-07T10:43:53,134 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-07T10:43:53,134 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-07T10:43:53,134 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-07T10:43:53,134 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-07T10:43:53,134 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-07T10:43:53,134 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-07T10:43:53,134 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-07T10:43:53,134 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-07T10:43:53,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7a68cc5b6ea7d9e2214ab42d65b16eba, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bcc3491e6200809e4c9c7d529da2c4cf, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dbe69e720cc4f2a4a4bc41171b6b3c32, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d4b381fbb1d4cd388f45774fca53b2eb, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3f6021bbda5b8b7b00318a57f02cd77d, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ddeeab6f2e44d85eb1ee916d3c5e8e9f, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=db89a463448eb182960960c83f4e0b16, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c3ab41a5354d389521b694050824788e, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f89575b73864cac8b7b164f417ef378f, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=72bf61b738a2f9b35913a192fc32815b, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0a7d18696a90cfed5af1fbdb2a706b65, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1baabb362104b46ecb78fe236391ca6f, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8ecb70a5eda60ab5744089e2ffe59d71, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9b830813c6b6c3ebcdb59061af10faad, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dec39ef6d94f3a519151705fce17adc4, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=541874b6b96014b10121ede508f1f61f, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=64da674e3a9ec8e7458eb07874d9f204, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9fba8e9ebafee4e67bfa1295ac44d4a5, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1701fae2bf725fdd94c16d1c6b8bc75f, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2cf36f48b84f67b77887428ec6ad5b31, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=69160268ea309aa55ece0cd8528b941d, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=257c2c76c61c100547acb4a778fb1fef, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1dd5c615432f6d3976b7d3e591d0a96a, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0b29f5d988381aa77506fa3b92944854, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8f449585c6b7b82f4442604100a49644, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c71057666e459779c2fa5a4311f4e99c, ASSIGN}] 2024-11-07T10:43:53,149 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c71057666e459779c2fa5a4311f4e99c, ASSIGN 2024-11-07T10:43:53,151 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c71057666e459779c2fa5a4311f4e99c, ASSIGN; state=OFFLINE, location=10641b2c0a69,40573,1730976230476; forceNewPlan=false, retain=false 2024-11-07T10:43:53,153 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8f449585c6b7b82f4442604100a49644, ASSIGN 2024-11-07T10:43:53,154 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8f449585c6b7b82f4442604100a49644, ASSIGN; state=OFFLINE, location=10641b2c0a69,41401,1730976230578; forceNewPlan=false, retain=false 2024-11-07T10:43:53,155 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0b29f5d988381aa77506fa3b92944854, ASSIGN 2024-11-07T10:43:53,156 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0b29f5d988381aa77506fa3b92944854, ASSIGN; state=OFFLINE, location=10641b2c0a69,39013,1730976230620; forceNewPlan=false, retain=false 2024-11-07T10:43:53,158 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1dd5c615432f6d3976b7d3e591d0a96a, ASSIGN 2024-11-07T10:43:53,159 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1dd5c615432f6d3976b7d3e591d0a96a, ASSIGN; state=OFFLINE, location=10641b2c0a69,40573,1730976230476; forceNewPlan=false, retain=false 2024-11-07T10:43:53,160 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=257c2c76c61c100547acb4a778fb1fef, ASSIGN 2024-11-07T10:43:53,162 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=257c2c76c61c100547acb4a778fb1fef, ASSIGN; state=OFFLINE, location=10641b2c0a69,41401,1730976230578; forceNewPlan=false, retain=false 2024-11-07T10:43:53,163 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=69160268ea309aa55ece0cd8528b941d, ASSIGN 2024-11-07T10:43:53,164 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=69160268ea309aa55ece0cd8528b941d, ASSIGN; state=OFFLINE, location=10641b2c0a69,39013,1730976230620; forceNewPlan=false, retain=false 2024-11-07T10:43:53,165 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2cf36f48b84f67b77887428ec6ad5b31, ASSIGN 2024-11-07T10:43:53,167 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2cf36f48b84f67b77887428ec6ad5b31, ASSIGN; state=OFFLINE, location=10641b2c0a69,40573,1730976230476; forceNewPlan=false, retain=false 2024-11-07T10:43:53,168 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1701fae2bf725fdd94c16d1c6b8bc75f, ASSIGN 2024-11-07T10:43:53,169 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1701fae2bf725fdd94c16d1c6b8bc75f, ASSIGN; state=OFFLINE, location=10641b2c0a69,41401,1730976230578; forceNewPlan=false, retain=false 2024-11-07T10:43:53,170 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9fba8e9ebafee4e67bfa1295ac44d4a5, ASSIGN 2024-11-07T10:43:53,172 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9fba8e9ebafee4e67bfa1295ac44d4a5, ASSIGN; state=OFFLINE, location=10641b2c0a69,40573,1730976230476; forceNewPlan=false, retain=false 2024-11-07T10:43:53,173 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=64da674e3a9ec8e7458eb07874d9f204, ASSIGN 2024-11-07T10:43:53,174 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=64da674e3a9ec8e7458eb07874d9f204, ASSIGN; state=OFFLINE, location=10641b2c0a69,39013,1730976230620; forceNewPlan=false, retain=false 2024-11-07T10:43:53,176 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=541874b6b96014b10121ede508f1f61f, ASSIGN 2024-11-07T10:43:53,177 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=541874b6b96014b10121ede508f1f61f, ASSIGN; state=OFFLINE, location=10641b2c0a69,41401,1730976230578; forceNewPlan=false, retain=false 2024-11-07T10:43:53,178 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dec39ef6d94f3a519151705fce17adc4, ASSIGN 2024-11-07T10:43:53,179 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dec39ef6d94f3a519151705fce17adc4, ASSIGN; state=OFFLINE, location=10641b2c0a69,39013,1730976230620; forceNewPlan=false, retain=false 2024-11-07T10:43:53,180 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9b830813c6b6c3ebcdb59061af10faad, ASSIGN 2024-11-07T10:43:53,182 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9b830813c6b6c3ebcdb59061af10faad, ASSIGN; state=OFFLINE, location=10641b2c0a69,40573,1730976230476; forceNewPlan=false, retain=false 2024-11-07T10:43:53,183 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8ecb70a5eda60ab5744089e2ffe59d71, ASSIGN 2024-11-07T10:43:53,184 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8ecb70a5eda60ab5744089e2ffe59d71, ASSIGN; state=OFFLINE, location=10641b2c0a69,41401,1730976230578; forceNewPlan=false, retain=false 2024-11-07T10:43:53,185 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1baabb362104b46ecb78fe236391ca6f, ASSIGN 2024-11-07T10:43:53,186 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1baabb362104b46ecb78fe236391ca6f, ASSIGN; state=OFFLINE, location=10641b2c0a69,39013,1730976230620; forceNewPlan=false, retain=false 2024-11-07T10:43:53,188 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0a7d18696a90cfed5af1fbdb2a706b65, ASSIGN 2024-11-07T10:43:53,189 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0a7d18696a90cfed5af1fbdb2a706b65, ASSIGN; state=OFFLINE, location=10641b2c0a69,40573,1730976230476; forceNewPlan=false, retain=false 2024-11-07T10:43:53,190 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=72bf61b738a2f9b35913a192fc32815b, ASSIGN 2024-11-07T10:43:53,192 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=72bf61b738a2f9b35913a192fc32815b, ASSIGN; state=OFFLINE, location=10641b2c0a69,41401,1730976230578; forceNewPlan=false, retain=false 2024-11-07T10:43:53,193 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f89575b73864cac8b7b164f417ef378f, ASSIGN 2024-11-07T10:43:53,194 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f89575b73864cac8b7b164f417ef378f, ASSIGN; state=OFFLINE, location=10641b2c0a69,39013,1730976230620; forceNewPlan=false, retain=false 2024-11-07T10:43:53,196 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c3ab41a5354d389521b694050824788e, ASSIGN 2024-11-07T10:43:53,197 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c3ab41a5354d389521b694050824788e, ASSIGN; state=OFFLINE, location=10641b2c0a69,40573,1730976230476; forceNewPlan=false, retain=false 2024-11-07T10:43:53,199 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=db89a463448eb182960960c83f4e0b16, ASSIGN 2024-11-07T10:43:53,201 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=db89a463448eb182960960c83f4e0b16, ASSIGN; state=OFFLINE, location=10641b2c0a69,41401,1730976230578; forceNewPlan=false, retain=false 2024-11-07T10:43:53,202 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ddeeab6f2e44d85eb1ee916d3c5e8e9f, ASSIGN 2024-11-07T10:43:53,204 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ddeeab6f2e44d85eb1ee916d3c5e8e9f, ASSIGN; state=OFFLINE, location=10641b2c0a69,40573,1730976230476; forceNewPlan=false, retain=false 2024-11-07T10:43:53,206 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3f6021bbda5b8b7b00318a57f02cd77d, ASSIGN 2024-11-07T10:43:53,207 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3f6021bbda5b8b7b00318a57f02cd77d, ASSIGN; state=OFFLINE, location=10641b2c0a69,39013,1730976230620; forceNewPlan=false, retain=false 2024-11-07T10:43:53,209 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d4b381fbb1d4cd388f45774fca53b2eb, ASSIGN 2024-11-07T10:43:53,210 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d4b381fbb1d4cd388f45774fca53b2eb, ASSIGN; state=OFFLINE, location=10641b2c0a69,41401,1730976230578; forceNewPlan=false, retain=false 2024-11-07T10:43:53,212 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dbe69e720cc4f2a4a4bc41171b6b3c32, ASSIGN 2024-11-07T10:43:53,215 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dbe69e720cc4f2a4a4bc41171b6b3c32, ASSIGN; state=OFFLINE, location=10641b2c0a69,39013,1730976230620; forceNewPlan=false, retain=false 2024-11-07T10:43:53,216 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bcc3491e6200809e4c9c7d529da2c4cf, ASSIGN 2024-11-07T10:43:53,218 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bcc3491e6200809e4c9c7d529da2c4cf, ASSIGN; state=OFFLINE, location=10641b2c0a69,40573,1730976230476; forceNewPlan=false, retain=false 2024-11-07T10:43:53,219 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7a68cc5b6ea7d9e2214ab42d65b16eba, ASSIGN 2024-11-07T10:43:53,220 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7a68cc5b6ea7d9e2214ab42d65b16eba, ASSIGN; state=OFFLINE, location=10641b2c0a69,41401,1730976230578; forceNewPlan=false, retain=false 2024-11-07T10:43:53,304 INFO [10641b2c0a69:43605 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-07T10:43:53,305 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=8f449585c6b7b82f4442604100a49644, regionState=OPENING, regionLocation=10641b2c0a69,41401,1730976230578 2024-11-07T10:43:53,306 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=257c2c76c61c100547acb4a778fb1fef, regionState=OPENING, regionLocation=10641b2c0a69,41401,1730976230578 2024-11-07T10:43:53,307 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=1701fae2bf725fdd94c16d1c6b8bc75f, regionState=OPENING, regionLocation=10641b2c0a69,41401,1730976230578 2024-11-07T10:43:53,307 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=541874b6b96014b10121ede508f1f61f, regionState=OPENING, regionLocation=10641b2c0a69,41401,1730976230578 2024-11-07T10:43:53,308 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=8ecb70a5eda60ab5744089e2ffe59d71, regionState=OPENING, regionLocation=10641b2c0a69,41401,1730976230578 2024-11-07T10:43:53,308 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=72bf61b738a2f9b35913a192fc32815b, regionState=OPENING, regionLocation=10641b2c0a69,41401,1730976230578 2024-11-07T10:43:53,309 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=db89a463448eb182960960c83f4e0b16, regionState=OPENING, regionLocation=10641b2c0a69,41401,1730976230578 2024-11-07T10:43:53,310 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=d4b381fbb1d4cd388f45774fca53b2eb, regionState=OPENING, regionLocation=10641b2c0a69,41401,1730976230578 2024-11-07T10:43:53,311 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7a68cc5b6ea7d9e2214ab42d65b16eba, regionState=OPENING, regionLocation=10641b2c0a69,41401,1730976230578 2024-11-07T10:43:53,311 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8f449585c6b7b82f4442604100a49644, ASSIGN because future has completed 2024-11-07T10:43:53,312 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=c71057666e459779c2fa5a4311f4e99c, regionState=OPENING, regionLocation=10641b2c0a69,40573,1730976230476 2024-11-07T10:43:53,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f449585c6b7b82f4442604100a49644, server=10641b2c0a69,41401,1730976230578}] 2024-11-07T10:43:53,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=257c2c76c61c100547acb4a778fb1fef, ASSIGN because future has completed 2024-11-07T10:43:53,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1701fae2bf725fdd94c16d1c6b8bc75f, ASSIGN because future has completed 2024-11-07T10:43:53,325 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=541874b6b96014b10121ede508f1f61f, ASSIGN because future has completed 2024-11-07T10:43:53,327 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8ecb70a5eda60ab5744089e2ffe59d71, ASSIGN because future has completed 2024-11-07T10:43:53,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 541874b6b96014b10121ede508f1f61f, server=10641b2c0a69,41401,1730976230578}] 2024-11-07T10:43:53,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=72bf61b738a2f9b35913a192fc32815b, ASSIGN because future has completed 2024-11-07T10:43:53,335 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=db89a463448eb182960960c83f4e0b16, ASSIGN because future has completed 2024-11-07T10:43:53,337 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d4b381fbb1d4cd388f45774fca53b2eb, ASSIGN because future has completed 2024-11-07T10:43:53,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure d4b381fbb1d4cd388f45774fca53b2eb, server=10641b2c0a69,41401,1730976230578}] 2024-11-07T10:43:53,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7a68cc5b6ea7d9e2214ab42d65b16eba, ASSIGN because future has completed 2024-11-07T10:43:53,346 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c71057666e459779c2fa5a4311f4e99c, ASSIGN because future has completed 2024-11-07T10:43:53,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure c71057666e459779c2fa5a4311f4e99c, server=10641b2c0a69,40573,1730976230476}] 2024-11-07T10:43:53,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7a68cc5b6ea7d9e2214ab42d65b16eba, server=10641b2c0a69,41401,1730976230578}] 2024-11-07T10:43:53,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure db89a463448eb182960960c83f4e0b16, server=10641b2c0a69,41401,1730976230578}] 2024-11-07T10:43:53,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 72bf61b738a2f9b35913a192fc32815b, server=10641b2c0a69,41401,1730976230578}] 2024-11-07T10:43:53,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ecb70a5eda60ab5744089e2ffe59d71, server=10641b2c0a69,41401,1730976230578}] 2024-11-07T10:43:53,376 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1701fae2bf725fdd94c16d1c6b8bc75f, server=10641b2c0a69,41401,1730976230578}] 2024-11-07T10:43:53,381 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 257c2c76c61c100547acb4a778fb1fef, server=10641b2c0a69,41401,1730976230578}] 2024-11-07T10:43:53,385 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=1dd5c615432f6d3976b7d3e591d0a96a, regionState=OPENING, regionLocation=10641b2c0a69,40573,1730976230476 2024-11-07T10:43:53,386 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=2cf36f48b84f67b77887428ec6ad5b31, regionState=OPENING, regionLocation=10641b2c0a69,40573,1730976230476 2024-11-07T10:43:53,387 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=9fba8e9ebafee4e67bfa1295ac44d4a5, regionState=OPENING, regionLocation=10641b2c0a69,40573,1730976230476 2024-11-07T10:43:53,387 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=9b830813c6b6c3ebcdb59061af10faad, regionState=OPENING, regionLocation=10641b2c0a69,40573,1730976230476 2024-11-07T10:43:53,388 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=0a7d18696a90cfed5af1fbdb2a706b65, regionState=OPENING, regionLocation=10641b2c0a69,40573,1730976230476 2024-11-07T10:43:53,388 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=c3ab41a5354d389521b694050824788e, regionState=OPENING, regionLocation=10641b2c0a69,40573,1730976230476 2024-11-07T10:43:53,388 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1dd5c615432f6d3976b7d3e591d0a96a, ASSIGN because future has completed 2024-11-07T10:43:53,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1dd5c615432f6d3976b7d3e591d0a96a, server=10641b2c0a69,40573,1730976230476}] 2024-11-07T10:43:53,390 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2cf36f48b84f67b77887428ec6ad5b31, ASSIGN because future has completed 2024-11-07T10:43:53,392 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9fba8e9ebafee4e67bfa1295ac44d4a5, ASSIGN because future has completed 2024-11-07T10:43:53,393 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9b830813c6b6c3ebcdb59061af10faad, ASSIGN because future has completed 2024-11-07T10:43:53,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0a7d18696a90cfed5af1fbdb2a706b65, ASSIGN because future has completed 2024-11-07T10:43:53,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0a7d18696a90cfed5af1fbdb2a706b65, server=10641b2c0a69,40573,1730976230476}] 2024-11-07T10:43:53,397 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c3ab41a5354d389521b694050824788e, ASSIGN because future has completed 2024-11-07T10:43:53,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3ab41a5354d389521b694050824788e, server=10641b2c0a69,40573,1730976230476}] 2024-11-07T10:43:53,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b830813c6b6c3ebcdb59061af10faad, server=10641b2c0a69,40573,1730976230476}] 2024-11-07T10:43:53,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9fba8e9ebafee4e67bfa1295ac44d4a5, server=10641b2c0a69,40573,1730976230476}] 2024-11-07T10:43:53,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2cf36f48b84f67b77887428ec6ad5b31, server=10641b2c0a69,40573,1730976230476}] 2024-11-07T10:43:53,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T10:43:53,420 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=ddeeab6f2e44d85eb1ee916d3c5e8e9f, regionState=OPENING, regionLocation=10641b2c0a69,40573,1730976230476 2024-11-07T10:43:53,421 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=bcc3491e6200809e4c9c7d529da2c4cf, regionState=OPENING, regionLocation=10641b2c0a69,40573,1730976230476 2024-11-07T10:43:53,422 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=0b29f5d988381aa77506fa3b92944854, regionState=OPENING, regionLocation=10641b2c0a69,39013,1730976230620 2024-11-07T10:43:53,422 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=69160268ea309aa55ece0cd8528b941d, regionState=OPENING, regionLocation=10641b2c0a69,39013,1730976230620 2024-11-07T10:43:53,423 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=64da674e3a9ec8e7458eb07874d9f204, regionState=OPENING, regionLocation=10641b2c0a69,39013,1730976230620 2024-11-07T10:43:53,423 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=dec39ef6d94f3a519151705fce17adc4, regionState=OPENING, regionLocation=10641b2c0a69,39013,1730976230620 2024-11-07T10:43:53,423 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ddeeab6f2e44d85eb1ee916d3c5e8e9f, ASSIGN because future has completed 2024-11-07T10:43:53,424 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure ddeeab6f2e44d85eb1ee916d3c5e8e9f, server=10641b2c0a69,40573,1730976230476}] 2024-11-07T10:43:53,425 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bcc3491e6200809e4c9c7d529da2c4cf, ASSIGN because future has completed 2024-11-07T10:43:53,427 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0b29f5d988381aa77506fa3b92944854, ASSIGN because future has completed 2024-11-07T10:43:53,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=69160268ea309aa55ece0cd8528b941d, ASSIGN because future has completed 2024-11-07T10:43:53,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 69160268ea309aa55ece0cd8528b941d, server=10641b2c0a69,39013,1730976230620}] 2024-11-07T10:43:53,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=64da674e3a9ec8e7458eb07874d9f204, ASSIGN because future has completed 2024-11-07T10:43:53,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dec39ef6d94f3a519151705fce17adc4, ASSIGN because future has completed 2024-11-07T10:43:53,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure dec39ef6d94f3a519151705fce17adc4, server=10641b2c0a69,39013,1730976230620}] 2024-11-07T10:43:53,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64da674e3a9ec8e7458eb07874d9f204, server=10641b2c0a69,39013,1730976230620}] 2024-11-07T10:43:53,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b29f5d988381aa77506fa3b92944854, server=10641b2c0a69,39013,1730976230620}] 2024-11-07T10:43:53,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure bcc3491e6200809e4c9c7d529da2c4cf, server=10641b2c0a69,40573,1730976230476}] 2024-11-07T10:43:53,450 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1baabb362104b46ecb78fe236391ca6f, regionState=OPENING, regionLocation=10641b2c0a69,39013,1730976230620 2024-11-07T10:43:53,451 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=f89575b73864cac8b7b164f417ef378f, regionState=OPENING, regionLocation=10641b2c0a69,39013,1730976230620 2024-11-07T10:43:53,451 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=3f6021bbda5b8b7b00318a57f02cd77d, regionState=OPENING, regionLocation=10641b2c0a69,39013,1730976230620 2024-11-07T10:43:53,452 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=dbe69e720cc4f2a4a4bc41171b6b3c32, regionState=OPENING, regionLocation=10641b2c0a69,39013,1730976230620 2024-11-07T10:43:53,453 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1baabb362104b46ecb78fe236391ca6f, ASSIGN because future has completed 2024-11-07T10:43:53,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1baabb362104b46ecb78fe236391ca6f, server=10641b2c0a69,39013,1730976230620}] 2024-11-07T10:43:53,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f89575b73864cac8b7b164f417ef378f, ASSIGN because future has completed 2024-11-07T10:43:53,456 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3f6021bbda5b8b7b00318a57f02cd77d, ASSIGN because future has completed 2024-11-07T10:43:53,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3f6021bbda5b8b7b00318a57f02cd77d, server=10641b2c0a69,39013,1730976230620}] 2024-11-07T10:43:53,461 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dbe69e720cc4f2a4a4bc41171b6b3c32, ASSIGN because future has completed 2024-11-07T10:43:53,463 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure dbe69e720cc4f2a4a4bc41171b6b3c32, server=10641b2c0a69,39013,1730976230620}] 2024-11-07T10:43:53,467 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure f89575b73864cac8b7b164f417ef378f, server=10641b2c0a69,39013,1730976230620}] 2024-11-07T10:43:53,480 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb. 2024-11-07T10:43:53,481 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => d4b381fbb1d4cd388f45774fca53b2eb, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-07T10:43:53,481 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true d4b381fbb1d4cd388f45774fca53b2eb 2024-11-07T10:43:53,481 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:53,482 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for d4b381fbb1d4cd388f45774fca53b2eb 2024-11-07T10:43:53,482 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for d4b381fbb1d4cd388f45774fca53b2eb 2024-11-07T10:43:53,483 INFO [StoreOpener-d4b381fbb1d4cd388f45774fca53b2eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region d4b381fbb1d4cd388f45774fca53b2eb 2024-11-07T10:43:53,485 INFO [StoreOpener-d4b381fbb1d4cd388f45774fca53b2eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4b381fbb1d4cd388f45774fca53b2eb columnFamilyName colfamily11 2024-11-07T10:43:53,485 DEBUG [StoreOpener-d4b381fbb1d4cd388f45774fca53b2eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,486 INFO [StoreOpener-d4b381fbb1d4cd388f45774fca53b2eb-1 {}] regionserver.HStore(327): Store=d4b381fbb1d4cd388f45774fca53b2eb/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,486 INFO [StoreOpener-d4b381fbb1d4cd388f45774fca53b2eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region d4b381fbb1d4cd388f45774fca53b2eb 2024-11-07T10:43:53,488 INFO [StoreOpener-d4b381fbb1d4cd388f45774fca53b2eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4b381fbb1d4cd388f45774fca53b2eb columnFamilyName colfamily21 2024-11-07T10:43:53,488 DEBUG [StoreOpener-d4b381fbb1d4cd388f45774fca53b2eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,489 INFO [StoreOpener-d4b381fbb1d4cd388f45774fca53b2eb-1 {}] regionserver.HStore(327): Store=d4b381fbb1d4cd388f45774fca53b2eb/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,489 INFO [StoreOpener-d4b381fbb1d4cd388f45774fca53b2eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region d4b381fbb1d4cd388f45774fca53b2eb 2024-11-07T10:43:53,490 INFO [StoreOpener-d4b381fbb1d4cd388f45774fca53b2eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4b381fbb1d4cd388f45774fca53b2eb columnFamilyName colfamily31 2024-11-07T10:43:53,490 DEBUG [StoreOpener-d4b381fbb1d4cd388f45774fca53b2eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,491 INFO [StoreOpener-d4b381fbb1d4cd388f45774fca53b2eb-1 {}] regionserver.HStore(327): Store=d4b381fbb1d4cd388f45774fca53b2eb/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,491 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for d4b381fbb1d4cd388f45774fca53b2eb 2024-11-07T10:43:53,492 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d4b381fbb1d4cd388f45774fca53b2eb 2024-11-07T10:43:53,492 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d4b381fbb1d4cd388f45774fca53b2eb 2024-11-07T10:43:53,494 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for d4b381fbb1d4cd388f45774fca53b2eb 2024-11-07T10:43:53,494 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for d4b381fbb1d4cd388f45774fca53b2eb 2024-11-07T10:43:53,494 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:43:53,496 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for d4b381fbb1d4cd388f45774fca53b2eb 2024-11-07T10:43:53,499 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d4b381fbb1d4cd388f45774fca53b2eb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:43:53,499 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened d4b381fbb1d4cd388f45774fca53b2eb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63166265, jitterRate=-0.058749303221702576}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:43:53,499 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d4b381fbb1d4cd388f45774fca53b2eb 2024-11-07T10:43:53,500 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for d4b381fbb1d4cd388f45774fca53b2eb: Running coprocessor pre-open hook at 1730976233482Writing region info on filesystem at 1730976233482Initializing all the Stores at 1730976233483 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233483Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233483Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233483Cleaning up temporary data from old regions at 1730976233494 (+11 ms)Running coprocessor post-open hooks at 1730976233499 (+5 ms)Region opened successfully at 1730976233500 (+1 ms) 2024-11-07T10:43:53,502 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb., pid=33, masterSystemTime=1730976233474 2024-11-07T10:43:53,503 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-07T10:43:53,505 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51425, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-07T10:43:53,506 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb. 2024-11-07T10:43:53,506 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb. 2024-11-07T10:43:53,506 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba. 2024-11-07T10:43:53,506 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 7a68cc5b6ea7d9e2214ab42d65b16eba, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-07T10:43:53,507 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 7a68cc5b6ea7d9e2214ab42d65b16eba 2024-11-07T10:43:53,507 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:53,507 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 7a68cc5b6ea7d9e2214ab42d65b16eba 2024-11-07T10:43:53,507 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 7a68cc5b6ea7d9e2214ab42d65b16eba 2024-11-07T10:43:53,507 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=d4b381fbb1d4cd388f45774fca53b2eb, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,41401,1730976230578 2024-11-07T10:43:53,511 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure d4b381fbb1d4cd388f45774fca53b2eb, server=10641b2c0a69,41401,1730976230578 because future has completed 2024-11-07T10:43:53,512 INFO [StoreOpener-7a68cc5b6ea7d9e2214ab42d65b16eba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 7a68cc5b6ea7d9e2214ab42d65b16eba 2024-11-07T10:43:53,513 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1730976232769.c3ab41a5354d389521b694050824788e. 2024-11-07T10:43:53,513 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => c3ab41a5354d389521b694050824788e, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1730976232769.c3ab41a5354d389521b694050824788e.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-07T10:43:53,514 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true c3ab41a5354d389521b694050824788e 2024-11-07T10:43:53,514 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1730976232769.c3ab41a5354d389521b694050824788e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:53,514 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for c3ab41a5354d389521b694050824788e 2024-11-07T10:43:53,514 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for c3ab41a5354d389521b694050824788e 2024-11-07T10:43:53,516 INFO [StoreOpener-7a68cc5b6ea7d9e2214ab42d65b16eba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a68cc5b6ea7d9e2214ab42d65b16eba columnFamilyName colfamily11 2024-11-07T10:43:53,516 DEBUG [StoreOpener-7a68cc5b6ea7d9e2214ab42d65b16eba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,517 INFO [StoreOpener-c3ab41a5354d389521b694050824788e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region c3ab41a5354d389521b694050824788e 2024-11-07T10:43:53,517 INFO [StoreOpener-7a68cc5b6ea7d9e2214ab42d65b16eba-1 {}] regionserver.HStore(327): Store=7a68cc5b6ea7d9e2214ab42d65b16eba/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=8 2024-11-07T10:43:53,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure d4b381fbb1d4cd388f45774fca53b2eb, server=10641b2c0a69,41401,1730976230578 in 174 msec 2024-11-07T10:43:53,517 INFO [StoreOpener-7a68cc5b6ea7d9e2214ab42d65b16eba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 7a68cc5b6ea7d9e2214ab42d65b16eba 2024-11-07T10:43:53,519 INFO [StoreOpener-c3ab41a5354d389521b694050824788e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c3ab41a5354d389521b694050824788e columnFamilyName colfamily11 2024-11-07T10:43:53,519 DEBUG [StoreOpener-c3ab41a5354d389521b694050824788e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,519 INFO [StoreOpener-7a68cc5b6ea7d9e2214ab42d65b16eba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a68cc5b6ea7d9e2214ab42d65b16eba columnFamilyName colfamily21 2024-11-07T10:43:53,519 DEBUG [StoreOpener-7a68cc5b6ea7d9e2214ab42d65b16eba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,520 INFO [StoreOpener-c3ab41a5354d389521b694050824788e-1 {}] regionserver.HStore(327): Store=c3ab41a5354d389521b694050824788e/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,520 INFO [StoreOpener-c3ab41a5354d389521b694050824788e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region c3ab41a5354d389521b694050824788e 2024-11-07T10:43:53,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d4b381fbb1d4cd388f45774fca53b2eb, ASSIGN in 380 msec 2024-11-07T10:43:53,520 INFO [StoreOpener-7a68cc5b6ea7d9e2214ab42d65b16eba-1 {}] regionserver.HStore(327): Store=7a68cc5b6ea7d9e2214ab42d65b16eba/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,520 INFO [StoreOpener-7a68cc5b6ea7d9e2214ab42d65b16eba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 7a68cc5b6ea7d9e2214ab42d65b16eba 2024-11-07T10:43:53,522 INFO [StoreOpener-c3ab41a5354d389521b694050824788e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c3ab41a5354d389521b694050824788e columnFamilyName colfamily21 2024-11-07T10:43:53,522 DEBUG [StoreOpener-c3ab41a5354d389521b694050824788e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,522 INFO [StoreOpener-7a68cc5b6ea7d9e2214ab42d65b16eba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a68cc5b6ea7d9e2214ab42d65b16eba columnFamilyName colfamily31 2024-11-07T10:43:53,522 DEBUG [StoreOpener-7a68cc5b6ea7d9e2214ab42d65b16eba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,522 INFO [StoreOpener-c3ab41a5354d389521b694050824788e-1 {}] regionserver.HStore(327): Store=c3ab41a5354d389521b694050824788e/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,522 INFO [StoreOpener-c3ab41a5354d389521b694050824788e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region c3ab41a5354d389521b694050824788e 2024-11-07T10:43:53,523 INFO [StoreOpener-7a68cc5b6ea7d9e2214ab42d65b16eba-1 {}] regionserver.HStore(327): Store=7a68cc5b6ea7d9e2214ab42d65b16eba/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,523 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 7a68cc5b6ea7d9e2214ab42d65b16eba 2024-11-07T10:43:53,524 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7a68cc5b6ea7d9e2214ab42d65b16eba 2024-11-07T10:43:53,524 INFO [StoreOpener-c3ab41a5354d389521b694050824788e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c3ab41a5354d389521b694050824788e columnFamilyName colfamily31 2024-11-07T10:43:53,524 DEBUG [StoreOpener-c3ab41a5354d389521b694050824788e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,525 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7a68cc5b6ea7d9e2214ab42d65b16eba 2024-11-07T10:43:53,525 INFO [StoreOpener-c3ab41a5354d389521b694050824788e-1 {}] regionserver.HStore(327): Store=c3ab41a5354d389521b694050824788e/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,525 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for c3ab41a5354d389521b694050824788e 2024-11-07T10:43:53,527 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3ab41a5354d389521b694050824788e 2024-11-07T10:43:53,527 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 7a68cc5b6ea7d9e2214ab42d65b16eba 2024-11-07T10:43:53,528 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 7a68cc5b6ea7d9e2214ab42d65b16eba 2024-11-07T10:43:53,528 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3ab41a5354d389521b694050824788e 2024-11-07T10:43:53,528 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:43:53,529 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for c3ab41a5354d389521b694050824788e 2024-11-07T10:43:53,529 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for c3ab41a5354d389521b694050824788e 2024-11-07T10:43:53,530 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 7a68cc5b6ea7d9e2214ab42d65b16eba 2024-11-07T10:43:53,530 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:43:53,533 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for c3ab41a5354d389521b694050824788e 2024-11-07T10:43:53,533 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7a68cc5b6ea7d9e2214ab42d65b16eba/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:43:53,534 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 7a68cc5b6ea7d9e2214ab42d65b16eba; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62644583, jitterRate=-0.06652297079563141}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:43:53,534 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7a68cc5b6ea7d9e2214ab42d65b16eba 2024-11-07T10:43:53,534 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 7a68cc5b6ea7d9e2214ab42d65b16eba: Running coprocessor pre-open hook at 1730976233507Writing region info on filesystem at 1730976233507Initializing all the Stores at 1730976233509 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233509Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233512 (+3 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233512Cleaning up temporary data from old regions at 1730976233528 (+16 ms)Running coprocessor post-open hooks at 1730976233534 (+6 ms)Region opened successfully at 1730976233534 2024-11-07T10:43:53,536 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba., pid=35, masterSystemTime=1730976233474 2024-11-07T10:43:53,537 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3ab41a5354d389521b694050824788e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:43:53,537 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened c3ab41a5354d389521b694050824788e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70799589, jitterRate=0.05499608814716339}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:43:53,538 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c3ab41a5354d389521b694050824788e 2024-11-07T10:43:53,538 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for c3ab41a5354d389521b694050824788e: Running coprocessor pre-open hook at 1730976233514Writing region info on filesystem at 1730976233514Initializing all the Stores at 1730976233516 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233516Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233516Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233516Cleaning up temporary data from old regions at 1730976233529 (+13 ms)Running coprocessor post-open hooks at 1730976233538 (+9 ms)Region opened successfully at 1730976233538 2024-11-07T10:43:53,539 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba. 2024-11-07T10:43:53,539 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba. 2024-11-07T10:43:53,539 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644. 2024-11-07T10:43:53,540 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 8f449585c6b7b82f4442604100a49644, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-07T10:43:53,540 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1730976232769.c3ab41a5354d389521b694050824788e., pid=43, masterSystemTime=1730976233503 2024-11-07T10:43:53,540 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 8f449585c6b7b82f4442604100a49644 2024-11-07T10:43:53,540 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:53,540 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7a68cc5b6ea7d9e2214ab42d65b16eba, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,41401,1730976230578 2024-11-07T10:43:53,540 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 8f449585c6b7b82f4442604100a49644 2024-11-07T10:43:53,540 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 8f449585c6b7b82f4442604100a49644 2024-11-07T10:43:53,542 INFO [StoreOpener-8f449585c6b7b82f4442604100a49644-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 8f449585c6b7b82f4442604100a49644 2024-11-07T10:43:53,543 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1730976232769.c3ab41a5354d389521b694050824788e. 2024-11-07T10:43:53,543 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1730976232769.c3ab41a5354d389521b694050824788e. 2024-11-07T10:43:53,543 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1730976232769.9fba8e9ebafee4e67bfa1295ac44d4a5. 2024-11-07T10:43:53,543 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=c3ab41a5354d389521b694050824788e, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,40573,1730976230476 2024-11-07T10:43:53,543 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 9fba8e9ebafee4e67bfa1295ac44d4a5, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1730976232769.9fba8e9ebafee4e67bfa1295ac44d4a5.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-07T10:43:53,544 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 9fba8e9ebafee4e67bfa1295ac44d4a5 2024-11-07T10:43:53,544 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1730976232769.9fba8e9ebafee4e67bfa1295ac44d4a5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:53,544 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 9fba8e9ebafee4e67bfa1295ac44d4a5 2024-11-07T10:43:53,544 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 9fba8e9ebafee4e67bfa1295ac44d4a5 2024-11-07T10:43:53,545 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7a68cc5b6ea7d9e2214ab42d65b16eba, server=10641b2c0a69,41401,1730976230578 because future has completed 2024-11-07T10:43:53,546 INFO [StoreOpener-9fba8e9ebafee4e67bfa1295ac44d4a5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 9fba8e9ebafee4e67bfa1295ac44d4a5 2024-11-07T10:43:53,547 INFO [StoreOpener-8f449585c6b7b82f4442604100a49644-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f449585c6b7b82f4442604100a49644 columnFamilyName colfamily11 2024-11-07T10:43:53,547 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure c3ab41a5354d389521b694050824788e, server=10641b2c0a69,40573,1730976230476 because future has completed 2024-11-07T10:43:53,547 DEBUG [StoreOpener-8f449585c6b7b82f4442604100a49644-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,548 INFO [StoreOpener-8f449585c6b7b82f4442604100a49644-1 {}] regionserver.HStore(327): Store=8f449585c6b7b82f4442604100a49644/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,548 INFO [StoreOpener-8f449585c6b7b82f4442604100a49644-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 8f449585c6b7b82f4442604100a49644 2024-11-07T10:43:53,550 INFO [StoreOpener-8f449585c6b7b82f4442604100a49644-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f449585c6b7b82f4442604100a49644 columnFamilyName colfamily21 2024-11-07T10:43:53,550 DEBUG [StoreOpener-8f449585c6b7b82f4442604100a49644-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,550 INFO [StoreOpener-9fba8e9ebafee4e67bfa1295ac44d4a5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9fba8e9ebafee4e67bfa1295ac44d4a5 columnFamilyName colfamily11 2024-11-07T10:43:53,551 DEBUG [StoreOpener-9fba8e9ebafee4e67bfa1295ac44d4a5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,551 INFO [StoreOpener-8f449585c6b7b82f4442604100a49644-1 {}] regionserver.HStore(327): Store=8f449585c6b7b82f4442604100a49644/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,551 INFO [StoreOpener-8f449585c6b7b82f4442604100a49644-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 8f449585c6b7b82f4442604100a49644 2024-11-07T10:43:53,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=5 2024-11-07T10:43:53,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 7a68cc5b6ea7d9e2214ab42d65b16eba, server=10641b2c0a69,41401,1730976230578 in 192 msec 2024-11-07T10:43:53,552 INFO [StoreOpener-9fba8e9ebafee4e67bfa1295ac44d4a5-1 {}] regionserver.HStore(327): Store=9fba8e9ebafee4e67bfa1295ac44d4a5/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,552 INFO [StoreOpener-9fba8e9ebafee4e67bfa1295ac44d4a5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 9fba8e9ebafee4e67bfa1295ac44d4a5 2024-11-07T10:43:53,553 INFO [StoreOpener-8f449585c6b7b82f4442604100a49644-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f449585c6b7b82f4442604100a49644 columnFamilyName colfamily31 2024-11-07T10:43:53,553 DEBUG [StoreOpener-8f449585c6b7b82f4442604100a49644-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7a68cc5b6ea7d9e2214ab42d65b16eba, ASSIGN in 414 msec 2024-11-07T10:43:53,554 INFO [StoreOpener-9fba8e9ebafee4e67bfa1295ac44d4a5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9fba8e9ebafee4e67bfa1295ac44d4a5 columnFamilyName colfamily21 2024-11-07T10:43:53,554 DEBUG [StoreOpener-9fba8e9ebafee4e67bfa1295ac44d4a5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,553 INFO [StoreOpener-8f449585c6b7b82f4442604100a49644-1 {}] regionserver.HStore(327): Store=8f449585c6b7b82f4442604100a49644/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,555 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 8f449585c6b7b82f4442604100a49644 2024-11-07T10:43:53,555 INFO [StoreOpener-9fba8e9ebafee4e67bfa1295ac44d4a5-1 {}] regionserver.HStore(327): Store=9fba8e9ebafee4e67bfa1295ac44d4a5/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,555 INFO [StoreOpener-9fba8e9ebafee4e67bfa1295ac44d4a5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 9fba8e9ebafee4e67bfa1295ac44d4a5 2024-11-07T10:43:53,556 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8f449585c6b7b82f4442604100a49644 2024-11-07T10:43:53,556 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8f449585c6b7b82f4442604100a49644 2024-11-07T10:43:53,557 INFO [StoreOpener-9fba8e9ebafee4e67bfa1295ac44d4a5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9fba8e9ebafee4e67bfa1295ac44d4a5 columnFamilyName colfamily31 2024-11-07T10:43:53,557 DEBUG [StoreOpener-9fba8e9ebafee4e67bfa1295ac44d4a5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,558 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 8f449585c6b7b82f4442604100a49644 2024-11-07T10:43:53,558 INFO [StoreOpener-9fba8e9ebafee4e67bfa1295ac44d4a5-1 {}] regionserver.HStore(327): Store=9fba8e9ebafee4e67bfa1295ac44d4a5/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,558 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 8f449585c6b7b82f4442604100a49644 2024-11-07T10:43:53,558 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 9fba8e9ebafee4e67bfa1295ac44d4a5 2024-11-07T10:43:53,559 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9fba8e9ebafee4e67bfa1295ac44d4a5 2024-11-07T10:43:53,560 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:43:53,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=12 2024-11-07T10:43:53,560 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9fba8e9ebafee4e67bfa1295ac44d4a5 2024-11-07T10:43:53,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure c3ab41a5354d389521b694050824788e, server=10641b2c0a69,40573,1730976230476 in 153 msec 2024-11-07T10:43:53,562 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 9fba8e9ebafee4e67bfa1295ac44d4a5 2024-11-07T10:43:53,562 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 9fba8e9ebafee4e67bfa1295ac44d4a5 2024-11-07T10:43:53,562 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 8f449585c6b7b82f4442604100a49644 2024-11-07T10:43:53,563 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:43:53,564 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c3ab41a5354d389521b694050824788e, ASSIGN in 423 msec 2024-11-07T10:43:53,565 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 9fba8e9ebafee4e67bfa1295ac44d4a5 2024-11-07T10:43:53,565 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8f449585c6b7b82f4442604100a49644/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:43:53,566 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 8f449585c6b7b82f4442604100a49644; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69576596, jitterRate=0.036772072315216064}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:43:53,566 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8f449585c6b7b82f4442604100a49644 2024-11-07T10:43:53,567 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 8f449585c6b7b82f4442604100a49644: Running coprocessor pre-open hook at 1730976233541Writing region info on filesystem at 1730976233541Initializing all the Stores at 1730976233542 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233542Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233542Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233542Cleaning up temporary data from old regions at 1730976233558 (+16 ms)Running coprocessor post-open hooks at 1730976233566 (+8 ms)Region opened successfully at 1730976233566 2024-11-07T10:43:53,568 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644., pid=31, masterSystemTime=1730976233474 2024-11-07T10:43:53,568 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9fba8e9ebafee4e67bfa1295ac44d4a5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:43:53,569 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 9fba8e9ebafee4e67bfa1295ac44d4a5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67865376, jitterRate=0.011272907257080078}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:43:53,569 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9fba8e9ebafee4e67bfa1295ac44d4a5 2024-11-07T10:43:53,570 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 9fba8e9ebafee4e67bfa1295ac44d4a5: Running coprocessor pre-open hook at 1730976233544Writing region info on filesystem at 1730976233544Initializing all the Stores at 1730976233546 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233546Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233546Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233546Cleaning up temporary data from old regions at 1730976233562 (+16 ms)Running coprocessor post-open hooks at 1730976233569 (+7 ms)Region opened successfully at 1730976233569 2024-11-07T10:43:53,571 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1730976232769.9fba8e9ebafee4e67bfa1295ac44d4a5., pid=45, masterSystemTime=1730976233503 2024-11-07T10:43:53,571 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644. 2024-11-07T10:43:53,571 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644. 2024-11-07T10:43:53,572 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=8f449585c6b7b82f4442604100a49644, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,41401,1730976230578 2024-11-07T10:43:53,572 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16. 2024-11-07T10:43:53,573 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => db89a463448eb182960960c83f4e0b16, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-07T10:43:53,573 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true db89a463448eb182960960c83f4e0b16 2024-11-07T10:43:53,573 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:53,573 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for db89a463448eb182960960c83f4e0b16 2024-11-07T10:43:53,573 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for db89a463448eb182960960c83f4e0b16 2024-11-07T10:43:53,575 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1730976232769.9fba8e9ebafee4e67bfa1295ac44d4a5. 2024-11-07T10:43:53,575 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1730976232769.9fba8e9ebafee4e67bfa1295ac44d4a5. 2024-11-07T10:43:53,575 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1730976232769.ddeeab6f2e44d85eb1ee916d3c5e8e9f. 2024-11-07T10:43:53,576 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => ddeeab6f2e44d85eb1ee916d3c5e8e9f, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1730976232769.ddeeab6f2e44d85eb1ee916d3c5e8e9f.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-07T10:43:53,576 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=9fba8e9ebafee4e67bfa1295ac44d4a5, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,40573,1730976230476 2024-11-07T10:43:53,576 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true ddeeab6f2e44d85eb1ee916d3c5e8e9f 2024-11-07T10:43:53,576 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1730976232769.ddeeab6f2e44d85eb1ee916d3c5e8e9f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:53,576 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for ddeeab6f2e44d85eb1ee916d3c5e8e9f 2024-11-07T10:43:53,576 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for ddeeab6f2e44d85eb1ee916d3c5e8e9f 2024-11-07T10:43:53,576 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f449585c6b7b82f4442604100a49644, server=10641b2c0a69,41401,1730976230578 because future has completed 2024-11-07T10:43:53,579 INFO [StoreOpener-db89a463448eb182960960c83f4e0b16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region db89a463448eb182960960c83f4e0b16 2024-11-07T10:43:53,579 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9fba8e9ebafee4e67bfa1295ac44d4a5, server=10641b2c0a69,40573,1730976230476 because future has completed 2024-11-07T10:43:53,579 INFO [StoreOpener-ddeeab6f2e44d85eb1ee916d3c5e8e9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region ddeeab6f2e44d85eb1ee916d3c5e8e9f 2024-11-07T10:43:53,582 INFO [StoreOpener-ddeeab6f2e44d85eb1ee916d3c5e8e9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ddeeab6f2e44d85eb1ee916d3c5e8e9f columnFamilyName colfamily11 2024-11-07T10:43:53,582 INFO [StoreOpener-db89a463448eb182960960c83f4e0b16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db89a463448eb182960960c83f4e0b16 columnFamilyName colfamily11 2024-11-07T10:43:53,582 DEBUG [StoreOpener-ddeeab6f2e44d85eb1ee916d3c5e8e9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,582 DEBUG [StoreOpener-db89a463448eb182960960c83f4e0b16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,582 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-07T10:43:53,583 INFO [StoreOpener-db89a463448eb182960960c83f4e0b16-1 {}] regionserver.HStore(327): Store=db89a463448eb182960960c83f4e0b16/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,583 INFO [StoreOpener-db89a463448eb182960960c83f4e0b16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region db89a463448eb182960960c83f4e0b16 2024-11-07T10:43:53,583 INFO [StoreOpener-ddeeab6f2e44d85eb1ee916d3c5e8e9f-1 {}] regionserver.HStore(327): Store=ddeeab6f2e44d85eb1ee916d3c5e8e9f/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,584 INFO [StoreOpener-ddeeab6f2e44d85eb1ee916d3c5e8e9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region ddeeab6f2e44d85eb1ee916d3c5e8e9f 2024-11-07T10:43:53,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=29 2024-11-07T10:43:53,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 8f449585c6b7b82f4442604100a49644, server=10641b2c0a69,41401,1730976230578 in 268 msec 2024-11-07T10:43:53,585 INFO [StoreOpener-db89a463448eb182960960c83f4e0b16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db89a463448eb182960960c83f4e0b16 columnFamilyName colfamily21 2024-11-07T10:43:53,585 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45063, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-07T10:43:53,585 DEBUG [StoreOpener-db89a463448eb182960960c83f4e0b16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,586 INFO [StoreOpener-ddeeab6f2e44d85eb1ee916d3c5e8e9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ddeeab6f2e44d85eb1ee916d3c5e8e9f columnFamilyName colfamily21 2024-11-07T10:43:53,586 DEBUG [StoreOpener-ddeeab6f2e44d85eb1ee916d3c5e8e9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,586 INFO [StoreOpener-db89a463448eb182960960c83f4e0b16-1 {}] regionserver.HStore(327): Store=db89a463448eb182960960c83f4e0b16/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,586 INFO [StoreOpener-db89a463448eb182960960c83f4e0b16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region db89a463448eb182960960c83f4e0b16 2024-11-07T10:43:53,587 INFO [StoreOpener-ddeeab6f2e44d85eb1ee916d3c5e8e9f-1 {}] regionserver.HStore(327): Store=ddeeab6f2e44d85eb1ee916d3c5e8e9f/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,587 INFO [StoreOpener-ddeeab6f2e44d85eb1ee916d3c5e8e9f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region ddeeab6f2e44d85eb1ee916d3c5e8e9f 2024-11-07T10:43:53,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8f449585c6b7b82f4442604100a49644, ASSIGN in 448 msec 2024-11-07T10:43:53,589 INFO [StoreOpener-db89a463448eb182960960c83f4e0b16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db89a463448eb182960960c83f4e0b16 columnFamilyName colfamily31 2024-11-07T10:43:53,590 INFO [StoreOpener-ddeeab6f2e44d85eb1ee916d3c5e8e9f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ddeeab6f2e44d85eb1ee916d3c5e8e9f columnFamilyName colfamily31 2024-11-07T10:43:53,590 DEBUG [StoreOpener-ddeeab6f2e44d85eb1ee916d3c5e8e9f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,589 DEBUG [StoreOpener-db89a463448eb182960960c83f4e0b16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,591 INFO [StoreOpener-ddeeab6f2e44d85eb1ee916d3c5e8e9f-1 {}] regionserver.HStore(327): Store=ddeeab6f2e44d85eb1ee916d3c5e8e9f/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,591 INFO [StoreOpener-db89a463448eb182960960c83f4e0b16-1 {}] regionserver.HStore(327): Store=db89a463448eb182960960c83f4e0b16/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,591 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for ddeeab6f2e44d85eb1ee916d3c5e8e9f 2024-11-07T10:43:53,591 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for db89a463448eb182960960c83f4e0b16 2024-11-07T10:43:53,592 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ddeeab6f2e44d85eb1ee916d3c5e8e9f 2024-11-07T10:43:53,592 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/db89a463448eb182960960c83f4e0b16 2024-11-07T10:43:53,593 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/db89a463448eb182960960c83f4e0b16 2024-11-07T10:43:53,593 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ddeeab6f2e44d85eb1ee916d3c5e8e9f 2024-11-07T10:43:53,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=22 2024-11-07T10:43:53,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 9fba8e9ebafee4e67bfa1295ac44d4a5, server=10641b2c0a69,40573,1730976230476 in 179 msec 2024-11-07T10:43:53,595 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1730976232769.64da674e3a9ec8e7458eb07874d9f204. 2024-11-07T10:43:53,595 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 64da674e3a9ec8e7458eb07874d9f204, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1730976232769.64da674e3a9ec8e7458eb07874d9f204.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-07T10:43:53,595 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 64da674e3a9ec8e7458eb07874d9f204 2024-11-07T10:43:53,595 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for db89a463448eb182960960c83f4e0b16 2024-11-07T10:43:53,595 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1730976232769.64da674e3a9ec8e7458eb07874d9f204.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:53,596 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 64da674e3a9ec8e7458eb07874d9f204 2024-11-07T10:43:53,596 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 64da674e3a9ec8e7458eb07874d9f204 2024-11-07T10:43:53,596 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for db89a463448eb182960960c83f4e0b16 2024-11-07T10:43:53,596 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for ddeeab6f2e44d85eb1ee916d3c5e8e9f 2024-11-07T10:43:53,596 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for ddeeab6f2e44d85eb1ee916d3c5e8e9f 2024-11-07T10:43:53,597 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:43:53,597 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:43:53,598 INFO [StoreOpener-64da674e3a9ec8e7458eb07874d9f204-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 64da674e3a9ec8e7458eb07874d9f204 2024-11-07T10:43:53,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9fba8e9ebafee4e67bfa1295ac44d4a5, ASSIGN in 457 msec 2024-11-07T10:43:53,599 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for db89a463448eb182960960c83f4e0b16 2024-11-07T10:43:53,599 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for ddeeab6f2e44d85eb1ee916d3c5e8e9f 2024-11-07T10:43:53,600 INFO [StoreOpener-64da674e3a9ec8e7458eb07874d9f204-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64da674e3a9ec8e7458eb07874d9f204 columnFamilyName colfamily11 2024-11-07T10:43:53,601 DEBUG [StoreOpener-64da674e3a9ec8e7458eb07874d9f204-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,601 INFO [StoreOpener-64da674e3a9ec8e7458eb07874d9f204-1 {}] regionserver.HStore(327): Store=64da674e3a9ec8e7458eb07874d9f204/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,602 INFO [StoreOpener-64da674e3a9ec8e7458eb07874d9f204-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 64da674e3a9ec8e7458eb07874d9f204 2024-11-07T10:43:53,603 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/db89a463448eb182960960c83f4e0b16/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:43:53,604 INFO [StoreOpener-64da674e3a9ec8e7458eb07874d9f204-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64da674e3a9ec8e7458eb07874d9f204 columnFamilyName colfamily21 2024-11-07T10:43:53,604 DEBUG [StoreOpener-64da674e3a9ec8e7458eb07874d9f204-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,604 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened db89a463448eb182960960c83f4e0b16; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61924248, jitterRate=-0.07725679874420166}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:43:53,604 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for db89a463448eb182960960c83f4e0b16 2024-11-07T10:43:53,605 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for db89a463448eb182960960c83f4e0b16: Running coprocessor pre-open hook at 1730976233574Writing region info on filesystem at 1730976233574Initializing all the Stores at 1730976233579 (+5 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233579Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233579Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233579Cleaning up temporary data from old regions at 1730976233596 (+17 ms)Running coprocessor post-open hooks at 1730976233604 (+8 ms)Region opened successfully at 1730976233604 2024-11-07T10:43:53,605 INFO [StoreOpener-64da674e3a9ec8e7458eb07874d9f204-1 {}] regionserver.HStore(327): Store=64da674e3a9ec8e7458eb07874d9f204/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,605 INFO [StoreOpener-64da674e3a9ec8e7458eb07874d9f204-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 64da674e3a9ec8e7458eb07874d9f204 2024-11-07T10:43:53,606 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16., pid=36, masterSystemTime=1730976233474 2024-11-07T10:43:53,607 INFO [StoreOpener-64da674e3a9ec8e7458eb07874d9f204-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64da674e3a9ec8e7458eb07874d9f204 columnFamilyName colfamily31 2024-11-07T10:43:53,607 DEBUG [StoreOpener-64da674e3a9ec8e7458eb07874d9f204-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,607 INFO [StoreOpener-64da674e3a9ec8e7458eb07874d9f204-1 {}] regionserver.HStore(327): Store=64da674e3a9ec8e7458eb07874d9f204/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,608 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 64da674e3a9ec8e7458eb07874d9f204 2024-11-07T10:43:53,608 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ddeeab6f2e44d85eb1ee916d3c5e8e9f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:43:53,609 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64da674e3a9ec8e7458eb07874d9f204 2024-11-07T10:43:53,609 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened ddeeab6f2e44d85eb1ee916d3c5e8e9f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61924002, jitterRate=-0.07726046442985535}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:43:53,609 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ddeeab6f2e44d85eb1ee916d3c5e8e9f 2024-11-07T10:43:53,609 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64da674e3a9ec8e7458eb07874d9f204 2024-11-07T10:43:53,609 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16. 2024-11-07T10:43:53,609 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for ddeeab6f2e44d85eb1ee916d3c5e8e9f: Running coprocessor pre-open hook at 1730976233577Writing region info on filesystem at 1730976233577Initializing all the Stores at 1730976233579 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233579Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233579Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233579Cleaning up temporary data from old regions at 1730976233596 (+17 ms)Running coprocessor post-open hooks at 1730976233609 (+13 ms)Region opened successfully at 1730976233609 2024-11-07T10:43:53,609 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16. 2024-11-07T10:43:53,610 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef. 2024-11-07T10:43:53,610 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 257c2c76c61c100547acb4a778fb1fef, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-07T10:43:53,610 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 257c2c76c61c100547acb4a778fb1fef 2024-11-07T10:43:53,610 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:53,611 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 257c2c76c61c100547acb4a778fb1fef 2024-11-07T10:43:53,611 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 257c2c76c61c100547acb4a778fb1fef 2024-11-07T10:43:53,611 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1730976232769.ddeeab6f2e44d85eb1ee916d3c5e8e9f., pid=47, masterSystemTime=1730976233503 2024-11-07T10:43:53,611 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 64da674e3a9ec8e7458eb07874d9f204 2024-11-07T10:43:53,612 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 64da674e3a9ec8e7458eb07874d9f204 2024-11-07T10:43:53,613 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:43:53,613 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=db89a463448eb182960960c83f4e0b16, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,41401,1730976230578 2024-11-07T10:43:53,615 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1730976232769.ddeeab6f2e44d85eb1ee916d3c5e8e9f. 2024-11-07T10:43:53,615 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1730976232769.ddeeab6f2e44d85eb1ee916d3c5e8e9f. 2024-11-07T10:43:53,615 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1730976232769.9b830813c6b6c3ebcdb59061af10faad. 2024-11-07T10:43:53,616 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 64da674e3a9ec8e7458eb07874d9f204 2024-11-07T10:43:53,616 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 9b830813c6b6c3ebcdb59061af10faad, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1730976232769.9b830813c6b6c3ebcdb59061af10faad.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-07T10:43:53,616 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 9b830813c6b6c3ebcdb59061af10faad 2024-11-07T10:43:53,616 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1730976232769.9b830813c6b6c3ebcdb59061af10faad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:53,617 INFO [StoreOpener-257c2c76c61c100547acb4a778fb1fef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 257c2c76c61c100547acb4a778fb1fef 2024-11-07T10:43:53,617 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 9b830813c6b6c3ebcdb59061af10faad 2024-11-07T10:43:53,617 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 9b830813c6b6c3ebcdb59061af10faad 2024-11-07T10:43:53,617 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=ddeeab6f2e44d85eb1ee916d3c5e8e9f, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,40573,1730976230476 2024-11-07T10:43:53,618 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure db89a463448eb182960960c83f4e0b16, server=10641b2c0a69,41401,1730976230578 because future has completed 2024-11-07T10:43:53,620 INFO [StoreOpener-257c2c76c61c100547acb4a778fb1fef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 257c2c76c61c100547acb4a778fb1fef columnFamilyName colfamily11 2024-11-07T10:43:53,620 DEBUG [StoreOpener-257c2c76c61c100547acb4a778fb1fef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,621 INFO [StoreOpener-9b830813c6b6c3ebcdb59061af10faad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 9b830813c6b6c3ebcdb59061af10faad 2024-11-07T10:43:53,621 INFO [StoreOpener-257c2c76c61c100547acb4a778fb1fef-1 {}] regionserver.HStore(327): Store=257c2c76c61c100547acb4a778fb1fef/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,621 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64da674e3a9ec8e7458eb07874d9f204/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:43:53,622 INFO [StoreOpener-257c2c76c61c100547acb4a778fb1fef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 257c2c76c61c100547acb4a778fb1fef 2024-11-07T10:43:53,622 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure ddeeab6f2e44d85eb1ee916d3c5e8e9f, server=10641b2c0a69,40573,1730976230476 because future has completed 2024-11-07T10:43:53,622 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 64da674e3a9ec8e7458eb07874d9f204; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71613033, jitterRate=0.0671173483133316}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:43:53,622 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 64da674e3a9ec8e7458eb07874d9f204 2024-11-07T10:43:53,623 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 64da674e3a9ec8e7458eb07874d9f204: Running coprocessor pre-open hook at 1730976233596Writing region info on filesystem at 1730976233596Initializing all the Stores at 1730976233597 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233598 (+1 ms)Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233598Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233598Cleaning up temporary data from old regions at 1730976233612 (+14 ms)Running coprocessor post-open hooks at 1730976233622 (+10 ms)Region opened successfully at 1730976233623 (+1 ms) 2024-11-07T10:43:53,624 INFO [StoreOpener-257c2c76c61c100547acb4a778fb1fef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 257c2c76c61c100547acb4a778fb1fef columnFamilyName colfamily21 2024-11-07T10:43:53,624 DEBUG [StoreOpener-257c2c76c61c100547acb4a778fb1fef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,624 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1730976232769.64da674e3a9ec8e7458eb07874d9f204., pid=50, masterSystemTime=1730976233582 2024-11-07T10:43:53,625 INFO [StoreOpener-257c2c76c61c100547acb4a778fb1fef-1 {}] regionserver.HStore(327): Store=257c2c76c61c100547acb4a778fb1fef/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,625 INFO [StoreOpener-257c2c76c61c100547acb4a778fb1fef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 257c2c76c61c100547acb4a778fb1fef 2024-11-07T10:43:53,625 INFO [StoreOpener-9b830813c6b6c3ebcdb59061af10faad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b830813c6b6c3ebcdb59061af10faad columnFamilyName colfamily11 2024-11-07T10:43:53,625 DEBUG [StoreOpener-9b830813c6b6c3ebcdb59061af10faad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,626 INFO [StoreOpener-9b830813c6b6c3ebcdb59061af10faad-1 {}] regionserver.HStore(327): Store=9b830813c6b6c3ebcdb59061af10faad/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,627 INFO [StoreOpener-9b830813c6b6c3ebcdb59061af10faad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 9b830813c6b6c3ebcdb59061af10faad 2024-11-07T10:43:53,627 INFO [StoreOpener-257c2c76c61c100547acb4a778fb1fef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 257c2c76c61c100547acb4a778fb1fef columnFamilyName colfamily31 2024-11-07T10:43:53,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=11 2024-11-07T10:43:53,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure db89a463448eb182960960c83f4e0b16, server=10641b2c0a69,41401,1730976230578 in 261 msec 2024-11-07T10:43:53,628 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1730976232769.64da674e3a9ec8e7458eb07874d9f204. 2024-11-07T10:43:53,628 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1730976232769.64da674e3a9ec8e7458eb07874d9f204. 2024-11-07T10:43:53,627 DEBUG [StoreOpener-257c2c76c61c100547acb4a778fb1fef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,628 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1730976232769.dec39ef6d94f3a519151705fce17adc4. 2024-11-07T10:43:53,629 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => dec39ef6d94f3a519151705fce17adc4, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1730976232769.dec39ef6d94f3a519151705fce17adc4.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-07T10:43:53,629 INFO [StoreOpener-257c2c76c61c100547acb4a778fb1fef-1 {}] regionserver.HStore(327): Store=257c2c76c61c100547acb4a778fb1fef/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,629 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true dec39ef6d94f3a519151705fce17adc4 2024-11-07T10:43:53,629 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 257c2c76c61c100547acb4a778fb1fef 2024-11-07T10:43:53,629 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1730976232769.dec39ef6d94f3a519151705fce17adc4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:53,629 INFO [StoreOpener-9b830813c6b6c3ebcdb59061af10faad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b830813c6b6c3ebcdb59061af10faad columnFamilyName colfamily21 2024-11-07T10:43:53,629 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for dec39ef6d94f3a519151705fce17adc4 2024-11-07T10:43:53,629 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for dec39ef6d94f3a519151705fce17adc4 2024-11-07T10:43:53,629 DEBUG [StoreOpener-9b830813c6b6c3ebcdb59061af10faad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,630 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/257c2c76c61c100547acb4a778fb1fef 2024-11-07T10:43:53,630 INFO [StoreOpener-9b830813c6b6c3ebcdb59061af10faad-1 {}] regionserver.HStore(327): Store=9b830813c6b6c3ebcdb59061af10faad/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,631 INFO [StoreOpener-9b830813c6b6c3ebcdb59061af10faad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 9b830813c6b6c3ebcdb59061af10faad 2024-11-07T10:43:53,631 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/257c2c76c61c100547acb4a778fb1fef 2024-11-07T10:43:53,631 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43605 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=10641b2c0a69,40573,1730976230476, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ddeeab6f2e44d85eb1ee916d3c5e8e9f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-07T10:43:53,632 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=64da674e3a9ec8e7458eb07874d9f204, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,39013,1730976230620 2024-11-07T10:43:53,633 INFO [StoreOpener-9b830813c6b6c3ebcdb59061af10faad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b830813c6b6c3ebcdb59061af10faad columnFamilyName colfamily31 2024-11-07T10:43:53,633 DEBUG [StoreOpener-9b830813c6b6c3ebcdb59061af10faad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,633 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 257c2c76c61c100547acb4a778fb1fef 2024-11-07T10:43:53,633 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 257c2c76c61c100547acb4a778fb1fef 2024-11-07T10:43:53,634 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:43:53,636 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64da674e3a9ec8e7458eb07874d9f204, server=10641b2c0a69,39013,1730976230620 because future has completed 2024-11-07T10:43:53,636 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 257c2c76c61c100547acb4a778fb1fef 2024-11-07T10:43:53,637 INFO [StoreOpener-9b830813c6b6c3ebcdb59061af10faad-1 {}] regionserver.HStore(327): Store=9b830813c6b6c3ebcdb59061af10faad/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,637 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 9b830813c6b6c3ebcdb59061af10faad 2024-11-07T10:43:53,638 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9b830813c6b6c3ebcdb59061af10faad 2024-11-07T10:43:53,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=db89a463448eb182960960c83f4e0b16, ASSIGN in 498 msec 2024-11-07T10:43:53,638 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9b830813c6b6c3ebcdb59061af10faad 2024-11-07T10:43:53,640 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 9b830813c6b6c3ebcdb59061af10faad 2024-11-07T10:43:53,640 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 9b830813c6b6c3ebcdb59061af10faad 2024-11-07T10:43:53,641 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:43:53,642 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 9b830813c6b6c3ebcdb59061af10faad 2024-11-07T10:43:53,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=21 2024-11-07T10:43:53,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 64da674e3a9ec8e7458eb07874d9f204, server=10641b2c0a69,39013,1730976230620 in 201 msec 2024-11-07T10:43:53,645 INFO [StoreOpener-dec39ef6d94f3a519151705fce17adc4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region dec39ef6d94f3a519151705fce17adc4 2024-11-07T10:43:53,646 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9b830813c6b6c3ebcdb59061af10faad/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:43:53,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=64da674e3a9ec8e7458eb07874d9f204, ASSIGN in 506 msec 2024-11-07T10:43:53,647 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 9b830813c6b6c3ebcdb59061af10faad; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63183694, jitterRate=-0.058489590883255005}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:43:53,647 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9b830813c6b6c3ebcdb59061af10faad 2024-11-07T10:43:53,647 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 9b830813c6b6c3ebcdb59061af10faad: Running coprocessor pre-open hook at 1730976233617Writing region info on filesystem at 1730976233617Initializing all the Stores at 1730976233619 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233619Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233621 (+2 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233621Cleaning up temporary data from old regions at 1730976233640 (+19 ms)Running coprocessor post-open hooks at 1730976233647 (+7 ms)Region opened successfully at 1730976233647 2024-11-07T10:43:53,648 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1730976232769.9b830813c6b6c3ebcdb59061af10faad., pid=44, masterSystemTime=1730976233503 2024-11-07T10:43:53,648 INFO [StoreOpener-dec39ef6d94f3a519151705fce17adc4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dec39ef6d94f3a519151705fce17adc4 columnFamilyName colfamily11 2024-11-07T10:43:53,648 DEBUG [StoreOpener-dec39ef6d94f3a519151705fce17adc4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,649 INFO [StoreOpener-dec39ef6d94f3a519151705fce17adc4-1 {}] regionserver.HStore(327): Store=dec39ef6d94f3a519151705fce17adc4/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,649 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/257c2c76c61c100547acb4a778fb1fef/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:43:53,649 INFO [StoreOpener-dec39ef6d94f3a519151705fce17adc4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region dec39ef6d94f3a519151705fce17adc4 2024-11-07T10:43:53,650 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 257c2c76c61c100547acb4a778fb1fef; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72768807, jitterRate=0.08433972299098969}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:43:53,650 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 257c2c76c61c100547acb4a778fb1fef 2024-11-07T10:43:53,650 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 257c2c76c61c100547acb4a778fb1fef: Running coprocessor pre-open hook at 1730976233611Writing region info on filesystem at 1730976233611Initializing all the Stores at 1730976233613 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233613Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233616 (+3 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233616Cleaning up temporary data from old regions at 1730976233633 (+17 ms)Running coprocessor post-open hooks at 1730976233650 (+17 ms)Region opened successfully at 1730976233650 2024-11-07T10:43:53,652 INFO [StoreOpener-dec39ef6d94f3a519151705fce17adc4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dec39ef6d94f3a519151705fce17adc4 columnFamilyName colfamily21 2024-11-07T10:43:53,652 DEBUG [StoreOpener-dec39ef6d94f3a519151705fce17adc4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,652 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef., pid=40, masterSystemTime=1730976233474 2024-11-07T10:43:53,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=10 2024-11-07T10:43:53,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure ddeeab6f2e44d85eb1ee916d3c5e8e9f, server=10641b2c0a69,40573,1730976230476 in 223 msec 2024-11-07T10:43:53,653 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1730976232769.9b830813c6b6c3ebcdb59061af10faad. 2024-11-07T10:43:53,653 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1730976232769.9b830813c6b6c3ebcdb59061af10faad. 2024-11-07T10:43:53,653 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1730976232769.1dd5c615432f6d3976b7d3e591d0a96a. 2024-11-07T10:43:53,653 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 1dd5c615432f6d3976b7d3e591d0a96a, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1730976232769.1dd5c615432f6d3976b7d3e591d0a96a.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-07T10:43:53,653 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 1dd5c615432f6d3976b7d3e591d0a96a 2024-11-07T10:43:53,654 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1730976232769.1dd5c615432f6d3976b7d3e591d0a96a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:53,654 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 1dd5c615432f6d3976b7d3e591d0a96a 2024-11-07T10:43:53,654 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 1dd5c615432f6d3976b7d3e591d0a96a 2024-11-07T10:43:53,654 INFO [StoreOpener-dec39ef6d94f3a519151705fce17adc4-1 {}] regionserver.HStore(327): Store=dec39ef6d94f3a519151705fce17adc4/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,654 INFO [StoreOpener-dec39ef6d94f3a519151705fce17adc4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region dec39ef6d94f3a519151705fce17adc4 2024-11-07T10:43:53,656 INFO [StoreOpener-1dd5c615432f6d3976b7d3e591d0a96a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 1dd5c615432f6d3976b7d3e591d0a96a 2024-11-07T10:43:53,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ddeeab6f2e44d85eb1ee916d3c5e8e9f, ASSIGN in 516 msec 2024-11-07T10:43:53,657 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef. 2024-11-07T10:43:53,657 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef. 2024-11-07T10:43:53,657 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f. 2024-11-07T10:43:53,657 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 541874b6b96014b10121ede508f1f61f, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-07T10:43:53,657 INFO [StoreOpener-dec39ef6d94f3a519151705fce17adc4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dec39ef6d94f3a519151705fce17adc4 columnFamilyName colfamily31 2024-11-07T10:43:53,658 DEBUG [StoreOpener-dec39ef6d94f3a519151705fce17adc4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,658 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=257c2c76c61c100547acb4a778fb1fef, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,41401,1730976230578 2024-11-07T10:43:53,658 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 541874b6b96014b10121ede508f1f61f 2024-11-07T10:43:53,658 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:53,658 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 541874b6b96014b10121ede508f1f61f 2024-11-07T10:43:53,658 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 541874b6b96014b10121ede508f1f61f 2024-11-07T10:43:53,660 INFO [StoreOpener-dec39ef6d94f3a519151705fce17adc4-1 {}] regionserver.HStore(327): Store=dec39ef6d94f3a519151705fce17adc4/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,660 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for dec39ef6d94f3a519151705fce17adc4 2024-11-07T10:43:53,661 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 257c2c76c61c100547acb4a778fb1fef, server=10641b2c0a69,41401,1730976230578 because future has completed 2024-11-07T10:43:53,662 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dec39ef6d94f3a519151705fce17adc4 2024-11-07T10:43:53,662 INFO [StoreOpener-1dd5c615432f6d3976b7d3e591d0a96a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1dd5c615432f6d3976b7d3e591d0a96a columnFamilyName colfamily11 2024-11-07T10:43:53,662 DEBUG [StoreOpener-1dd5c615432f6d3976b7d3e591d0a96a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,662 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=9b830813c6b6c3ebcdb59061af10faad, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,40573,1730976230476 2024-11-07T10:43:53,664 INFO [StoreOpener-541874b6b96014b10121ede508f1f61f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 541874b6b96014b10121ede508f1f61f 2024-11-07T10:43:53,667 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b830813c6b6c3ebcdb59061af10faad, server=10641b2c0a69,40573,1730976230476 because future has completed 2024-11-07T10:43:53,667 INFO [StoreOpener-1dd5c615432f6d3976b7d3e591d0a96a-1 {}] regionserver.HStore(327): Store=1dd5c615432f6d3976b7d3e591d0a96a/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,668 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dec39ef6d94f3a519151705fce17adc4 2024-11-07T10:43:53,668 INFO [StoreOpener-1dd5c615432f6d3976b7d3e591d0a96a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 1dd5c615432f6d3976b7d3e591d0a96a 2024-11-07T10:43:53,669 INFO [StoreOpener-541874b6b96014b10121ede508f1f61f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 541874b6b96014b10121ede508f1f61f columnFamilyName colfamily11 2024-11-07T10:43:53,669 DEBUG [StoreOpener-541874b6b96014b10121ede508f1f61f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,670 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43605 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=10641b2c0a69,41401,1730976230578, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=257c2c76c61c100547acb4a778fb1fef. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-07T10:43:53,670 INFO [StoreOpener-541874b6b96014b10121ede508f1f61f-1 {}] regionserver.HStore(327): Store=541874b6b96014b10121ede508f1f61f/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,670 INFO [StoreOpener-541874b6b96014b10121ede508f1f61f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 541874b6b96014b10121ede508f1f61f 2024-11-07T10:43:53,671 INFO [StoreOpener-1dd5c615432f6d3976b7d3e591d0a96a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1dd5c615432f6d3976b7d3e591d0a96a columnFamilyName colfamily21 2024-11-07T10:43:53,671 DEBUG [StoreOpener-1dd5c615432f6d3976b7d3e591d0a96a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,672 INFO [StoreOpener-1dd5c615432f6d3976b7d3e591d0a96a-1 {}] regionserver.HStore(327): Store=1dd5c615432f6d3976b7d3e591d0a96a/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,672 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for dec39ef6d94f3a519151705fce17adc4 2024-11-07T10:43:53,672 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for dec39ef6d94f3a519151705fce17adc4 2024-11-07T10:43:53,672 INFO [StoreOpener-1dd5c615432f6d3976b7d3e591d0a96a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 1dd5c615432f6d3976b7d3e591d0a96a 2024-11-07T10:43:53,672 INFO [StoreOpener-541874b6b96014b10121ede508f1f61f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 541874b6b96014b10121ede508f1f61f columnFamilyName colfamily21 2024-11-07T10:43:53,672 DEBUG [StoreOpener-541874b6b96014b10121ede508f1f61f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,673 INFO [StoreOpener-541874b6b96014b10121ede508f1f61f-1 {}] regionserver.HStore(327): Store=541874b6b96014b10121ede508f1f61f/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,673 INFO [StoreOpener-541874b6b96014b10121ede508f1f61f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 541874b6b96014b10121ede508f1f61f 2024-11-07T10:43:53,674 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:43:53,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=26 2024-11-07T10:43:53,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 257c2c76c61c100547acb4a778fb1fef, server=10641b2c0a69,41401,1730976230578 in 289 msec 2024-11-07T10:43:53,674 INFO [StoreOpener-1dd5c615432f6d3976b7d3e591d0a96a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1dd5c615432f6d3976b7d3e591d0a96a columnFamilyName colfamily31 2024-11-07T10:43:53,674 DEBUG [StoreOpener-1dd5c615432f6d3976b7d3e591d0a96a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,675 INFO [StoreOpener-541874b6b96014b10121ede508f1f61f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 541874b6b96014b10121ede508f1f61f columnFamilyName colfamily31 2024-11-07T10:43:53,675 INFO [StoreOpener-1dd5c615432f6d3976b7d3e591d0a96a-1 {}] regionserver.HStore(327): Store=1dd5c615432f6d3976b7d3e591d0a96a/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,675 DEBUG [StoreOpener-541874b6b96014b10121ede508f1f61f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,675 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 1dd5c615432f6d3976b7d3e591d0a96a 2024-11-07T10:43:53,676 INFO [StoreOpener-541874b6b96014b10121ede508f1f61f-1 {}] regionserver.HStore(327): Store=541874b6b96014b10121ede508f1f61f/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,676 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 541874b6b96014b10121ede508f1f61f 2024-11-07T10:43:53,676 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1dd5c615432f6d3976b7d3e591d0a96a 2024-11-07T10:43:53,677 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for dec39ef6d94f3a519151705fce17adc4 2024-11-07T10:43:53,677 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1dd5c615432f6d3976b7d3e591d0a96a 2024-11-07T10:43:53,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=257c2c76c61c100547acb4a778fb1fef, ASSIGN in 537 msec 2024-11-07T10:43:53,678 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/541874b6b96014b10121ede508f1f61f 2024-11-07T10:43:53,678 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/541874b6b96014b10121ede508f1f61f 2024-11-07T10:43:53,679 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 1dd5c615432f6d3976b7d3e591d0a96a 2024-11-07T10:43:53,679 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 1dd5c615432f6d3976b7d3e591d0a96a 2024-11-07T10:43:53,680 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:43:53,681 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 541874b6b96014b10121ede508f1f61f 2024-11-07T10:43:53,681 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 541874b6b96014b10121ede508f1f61f 2024-11-07T10:43:53,681 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dec39ef6d94f3a519151705fce17adc4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:43:53,682 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened dec39ef6d94f3a519151705fce17adc4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62130268, jitterRate=-0.07418686151504517}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:43:53,682 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dec39ef6d94f3a519151705fce17adc4 2024-11-07T10:43:53,682 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for dec39ef6d94f3a519151705fce17adc4: Running coprocessor pre-open hook at 1730976233630Writing region info on filesystem at 1730976233630Initializing all the Stores at 1730976233632 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233632Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233644 (+12 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233645 (+1 ms)Cleaning up temporary data from old regions at 1730976233672 (+27 ms)Running coprocessor post-open hooks at 1730976233682 (+10 ms)Region opened successfully at 1730976233682 2024-11-07T10:43:53,683 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:43:53,683 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 1dd5c615432f6d3976b7d3e591d0a96a 2024-11-07T10:43:53,683 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=18 2024-11-07T10:43:53,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 9b830813c6b6c3ebcdb59061af10faad, server=10641b2c0a69,40573,1730976230476 in 272 msec 2024-11-07T10:43:53,684 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1730976232769.dec39ef6d94f3a519151705fce17adc4., pid=49, masterSystemTime=1730976233582 2024-11-07T10:43:53,686 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 541874b6b96014b10121ede508f1f61f 2024-11-07T10:43:53,687 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1730976232769.dec39ef6d94f3a519151705fce17adc4. 2024-11-07T10:43:53,687 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1730976232769.dec39ef6d94f3a519151705fce17adc4. 2024-11-07T10:43:53,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9b830813c6b6c3ebcdb59061af10faad, ASSIGN in 547 msec 2024-11-07T10:43:53,687 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1730976232769.dbe69e720cc4f2a4a4bc41171b6b3c32. 2024-11-07T10:43:53,687 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => dbe69e720cc4f2a4a4bc41171b6b3c32, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1730976232769.dbe69e720cc4f2a4a4bc41171b6b3c32.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-07T10:43:53,688 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true dbe69e720cc4f2a4a4bc41171b6b3c32 2024-11-07T10:43:53,688 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1730976232769.dbe69e720cc4f2a4a4bc41171b6b3c32.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:53,688 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for dbe69e720cc4f2a4a4bc41171b6b3c32 2024-11-07T10:43:53,688 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for dbe69e720cc4f2a4a4bc41171b6b3c32 2024-11-07T10:43:53,689 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1dd5c615432f6d3976b7d3e591d0a96a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:43:53,689 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=dec39ef6d94f3a519151705fce17adc4, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,39013,1730976230620 2024-11-07T10:43:53,689 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 1dd5c615432f6d3976b7d3e591d0a96a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60723772, jitterRate=-0.09514528512954712}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:43:53,690 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1dd5c615432f6d3976b7d3e591d0a96a 2024-11-07T10:43:53,690 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/541874b6b96014b10121ede508f1f61f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:43:53,690 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 1dd5c615432f6d3976b7d3e591d0a96a: Running coprocessor pre-open hook at 1730976233654Writing region info on filesystem at 1730976233654Initializing all the Stores at 1730976233655 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233655Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233656 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233656Cleaning up temporary data from old regions at 1730976233679 (+23 ms)Running coprocessor post-open hooks at 1730976233690 (+11 ms)Region opened successfully at 1730976233690 2024-11-07T10:43:53,692 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 541874b6b96014b10121ede508f1f61f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59297753, jitterRate=-0.11639462411403656}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:43:53,692 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 541874b6b96014b10121ede508f1f61f 2024-11-07T10:43:53,692 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 541874b6b96014b10121ede508f1f61f: Running coprocessor pre-open hook at 1730976233658Writing region info on filesystem at 1730976233658Initializing all the Stores at 1730976233661 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233661Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233664 (+3 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233664Cleaning up temporary data from old regions at 1730976233681 (+17 ms)Running coprocessor post-open hooks at 1730976233692 (+11 ms)Region opened successfully at 1730976233692 2024-11-07T10:43:53,692 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1730976232769.1dd5c615432f6d3976b7d3e591d0a96a., pid=41, masterSystemTime=1730976233503 2024-11-07T10:43:53,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure dec39ef6d94f3a519151705fce17adc4, server=10641b2c0a69,39013,1730976230620 because future has completed 2024-11-07T10:43:53,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=19 2024-11-07T10:43:53,698 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure dec39ef6d94f3a519151705fce17adc4, server=10641b2c0a69,39013,1730976230620 in 261 msec 2024-11-07T10:43:53,701 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f., pid=32, masterSystemTime=1730976233474 2024-11-07T10:43:53,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dec39ef6d94f3a519151705fce17adc4, ASSIGN in 561 msec 2024-11-07T10:43:53,704 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1730976232769.1dd5c615432f6d3976b7d3e591d0a96a. 2024-11-07T10:43:53,704 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1730976232769.1dd5c615432f6d3976b7d3e591d0a96a. 2024-11-07T10:43:53,704 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1730976232769.2cf36f48b84f67b77887428ec6ad5b31. 2024-11-07T10:43:53,705 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 2cf36f48b84f67b77887428ec6ad5b31, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1730976232769.2cf36f48b84f67b77887428ec6ad5b31.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-07T10:43:53,705 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2cf36f48b84f67b77887428ec6ad5b31 2024-11-07T10:43:53,705 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1730976232769.2cf36f48b84f67b77887428ec6ad5b31.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:53,706 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 2cf36f48b84f67b77887428ec6ad5b31 2024-11-07T10:43:53,706 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 2cf36f48b84f67b77887428ec6ad5b31 2024-11-07T10:43:53,707 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=1dd5c615432f6d3976b7d3e591d0a96a, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,40573,1730976230476 2024-11-07T10:43:53,709 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f. 2024-11-07T10:43:53,709 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f. 2024-11-07T10:43:53,709 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b. 2024-11-07T10:43:53,709 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 72bf61b738a2f9b35913a192fc32815b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-07T10:43:53,710 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 72bf61b738a2f9b35913a192fc32815b 2024-11-07T10:43:53,710 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:53,710 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 72bf61b738a2f9b35913a192fc32815b 2024-11-07T10:43:53,710 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 72bf61b738a2f9b35913a192fc32815b 2024-11-07T10:43:53,712 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1dd5c615432f6d3976b7d3e591d0a96a, server=10641b2c0a69,40573,1730976230476 because future has completed 2024-11-07T10:43:53,713 INFO [StoreOpener-dbe69e720cc4f2a4a4bc41171b6b3c32-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region dbe69e720cc4f2a4a4bc41171b6b3c32 2024-11-07T10:43:53,713 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=541874b6b96014b10121ede508f1f61f, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,41401,1730976230578 2024-11-07T10:43:53,715 INFO [StoreOpener-dbe69e720cc4f2a4a4bc41171b6b3c32-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbe69e720cc4f2a4a4bc41171b6b3c32 columnFamilyName colfamily11 2024-11-07T10:43:53,716 DEBUG [StoreOpener-dbe69e720cc4f2a4a4bc41171b6b3c32-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,716 INFO [StoreOpener-dbe69e720cc4f2a4a4bc41171b6b3c32-1 {}] regionserver.HStore(327): Store=dbe69e720cc4f2a4a4bc41171b6b3c32/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,716 INFO [StoreOpener-dbe69e720cc4f2a4a4bc41171b6b3c32-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region dbe69e720cc4f2a4a4bc41171b6b3c32 2024-11-07T10:43:53,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 541874b6b96014b10121ede508f1f61f, server=10641b2c0a69,41401,1730976230578 because future has completed 2024-11-07T10:43:53,718 INFO [StoreOpener-dbe69e720cc4f2a4a4bc41171b6b3c32-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbe69e720cc4f2a4a4bc41171b6b3c32 columnFamilyName colfamily21 2024-11-07T10:43:53,718 DEBUG [StoreOpener-dbe69e720cc4f2a4a4bc41171b6b3c32-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,719 INFO [StoreOpener-dbe69e720cc4f2a4a4bc41171b6b3c32-1 {}] regionserver.HStore(327): Store=dbe69e720cc4f2a4a4bc41171b6b3c32/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,719 INFO [StoreOpener-dbe69e720cc4f2a4a4bc41171b6b3c32-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region dbe69e720cc4f2a4a4bc41171b6b3c32 2024-11-07T10:43:53,721 INFO [StoreOpener-dbe69e720cc4f2a4a4bc41171b6b3c32-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbe69e720cc4f2a4a4bc41171b6b3c32 columnFamilyName colfamily31 2024-11-07T10:43:53,721 DEBUG [StoreOpener-dbe69e720cc4f2a4a4bc41171b6b3c32-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,721 INFO [StoreOpener-dbe69e720cc4f2a4a4bc41171b6b3c32-1 {}] regionserver.HStore(327): Store=dbe69e720cc4f2a4a4bc41171b6b3c32/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,722 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for dbe69e720cc4f2a4a4bc41171b6b3c32 2024-11-07T10:43:53,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=20 2024-11-07T10:43:53,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 541874b6b96014b10121ede508f1f61f, server=10641b2c0a69,41401,1730976230578 in 391 msec 2024-11-07T10:43:53,723 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dbe69e720cc4f2a4a4bc41171b6b3c32 2024-11-07T10:43:53,723 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dbe69e720cc4f2a4a4bc41171b6b3c32 2024-11-07T10:43:53,725 INFO [StoreOpener-2cf36f48b84f67b77887428ec6ad5b31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 2cf36f48b84f67b77887428ec6ad5b31 2024-11-07T10:43:53,725 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for dbe69e720cc4f2a4a4bc41171b6b3c32 2024-11-07T10:43:53,725 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for dbe69e720cc4f2a4a4bc41171b6b3c32 2024-11-07T10:43:53,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=541874b6b96014b10121ede508f1f61f, ASSIGN in 585 msec 2024-11-07T10:43:53,726 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:43:53,728 INFO [StoreOpener-2cf36f48b84f67b77887428ec6ad5b31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2cf36f48b84f67b77887428ec6ad5b31 columnFamilyName colfamily11 2024-11-07T10:43:53,728 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for dbe69e720cc4f2a4a4bc41171b6b3c32 2024-11-07T10:43:53,728 DEBUG [StoreOpener-2cf36f48b84f67b77887428ec6ad5b31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,729 INFO [StoreOpener-2cf36f48b84f67b77887428ec6ad5b31-1 {}] regionserver.HStore(327): Store=2cf36f48b84f67b77887428ec6ad5b31/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,730 INFO [StoreOpener-2cf36f48b84f67b77887428ec6ad5b31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 2cf36f48b84f67b77887428ec6ad5b31 2024-11-07T10:43:53,732 INFO [StoreOpener-2cf36f48b84f67b77887428ec6ad5b31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2cf36f48b84f67b77887428ec6ad5b31 columnFamilyName colfamily21 2024-11-07T10:43:53,732 DEBUG [StoreOpener-2cf36f48b84f67b77887428ec6ad5b31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,733 INFO [StoreOpener-2cf36f48b84f67b77887428ec6ad5b31-1 {}] regionserver.HStore(327): Store=2cf36f48b84f67b77887428ec6ad5b31/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,733 INFO [StoreOpener-2cf36f48b84f67b77887428ec6ad5b31-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 2cf36f48b84f67b77887428ec6ad5b31 2024-11-07T10:43:53,736 INFO [StoreOpener-2cf36f48b84f67b77887428ec6ad5b31-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2cf36f48b84f67b77887428ec6ad5b31 columnFamilyName colfamily31 2024-11-07T10:43:53,741 INFO [StoreOpener-72bf61b738a2f9b35913a192fc32815b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 72bf61b738a2f9b35913a192fc32815b 2024-11-07T10:43:53,742 DEBUG [StoreOpener-2cf36f48b84f67b77887428ec6ad5b31-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,743 INFO [StoreOpener-2cf36f48b84f67b77887428ec6ad5b31-1 {}] regionserver.HStore(327): Store=2cf36f48b84f67b77887428ec6ad5b31/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,744 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 2cf36f48b84f67b77887428ec6ad5b31 2024-11-07T10:43:53,744 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dbe69e720cc4f2a4a4bc41171b6b3c32/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:43:53,745 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cf36f48b84f67b77887428ec6ad5b31 2024-11-07T10:43:53,745 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened dbe69e720cc4f2a4a4bc41171b6b3c32; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61710610, jitterRate=-0.08044025301933289}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:43:53,745 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dbe69e720cc4f2a4a4bc41171b6b3c32 2024-11-07T10:43:53,745 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for dbe69e720cc4f2a4a4bc41171b6b3c32: Running coprocessor pre-open hook at 1730976233688Writing region info on filesystem at 1730976233688Initializing all the Stores at 1730976233701 (+13 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233701Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233712 (+11 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233712Cleaning up temporary data from old regions at 1730976233725 (+13 ms)Running coprocessor post-open hooks at 1730976233745 (+20 ms)Region opened successfully at 1730976233745 2024-11-07T10:43:53,745 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cf36f48b84f67b77887428ec6ad5b31 2024-11-07T10:43:53,746 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1730976232769.dbe69e720cc4f2a4a4bc41171b6b3c32., pid=55, masterSystemTime=1730976233582 2024-11-07T10:43:53,747 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 2cf36f48b84f67b77887428ec6ad5b31 2024-11-07T10:43:53,747 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 2cf36f48b84f67b77887428ec6ad5b31 2024-11-07T10:43:53,747 INFO [StoreOpener-72bf61b738a2f9b35913a192fc32815b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72bf61b738a2f9b35913a192fc32815b columnFamilyName colfamily11 2024-11-07T10:43:53,747 DEBUG [StoreOpener-72bf61b738a2f9b35913a192fc32815b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,748 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:43:53,749 INFO [StoreOpener-72bf61b738a2f9b35913a192fc32815b-1 {}] regionserver.HStore(327): Store=72bf61b738a2f9b35913a192fc32815b/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,749 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=27 2024-11-07T10:43:53,749 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 1dd5c615432f6d3976b7d3e591d0a96a, server=10641b2c0a69,40573,1730976230476 in 342 msec 2024-11-07T10:43:53,749 INFO [StoreOpener-72bf61b738a2f9b35913a192fc32815b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 72bf61b738a2f9b35913a192fc32815b 2024-11-07T10:43:53,750 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1730976232769.dbe69e720cc4f2a4a4bc41171b6b3c32. 2024-11-07T10:43:53,750 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1730976232769.dbe69e720cc4f2a4a4bc41171b6b3c32. 2024-11-07T10:43:53,750 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1730976232769.69160268ea309aa55ece0cd8528b941d. 2024-11-07T10:43:53,750 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 69160268ea309aa55ece0cd8528b941d, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1730976232769.69160268ea309aa55ece0cd8528b941d.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-07T10:43:53,751 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 69160268ea309aa55ece0cd8528b941d 2024-11-07T10:43:53,751 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1730976232769.69160268ea309aa55ece0cd8528b941d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:53,751 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 69160268ea309aa55ece0cd8528b941d 2024-11-07T10:43:53,751 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 69160268ea309aa55ece0cd8528b941d 2024-11-07T10:43:53,751 INFO [StoreOpener-72bf61b738a2f9b35913a192fc32815b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72bf61b738a2f9b35913a192fc32815b columnFamilyName colfamily21 2024-11-07T10:43:53,751 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 2cf36f48b84f67b77887428ec6ad5b31 2024-11-07T10:43:53,751 DEBUG [StoreOpener-72bf61b738a2f9b35913a192fc32815b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,752 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=dbe69e720cc4f2a4a4bc41171b6b3c32, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,39013,1730976230620 2024-11-07T10:43:53,753 INFO [StoreOpener-69160268ea309aa55ece0cd8528b941d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 69160268ea309aa55ece0cd8528b941d 2024-11-07T10:43:53,753 INFO [StoreOpener-72bf61b738a2f9b35913a192fc32815b-1 {}] regionserver.HStore(327): Store=72bf61b738a2f9b35913a192fc32815b/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,754 INFO [StoreOpener-72bf61b738a2f9b35913a192fc32815b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 72bf61b738a2f9b35913a192fc32815b 2024-11-07T10:43:53,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure dbe69e720cc4f2a4a4bc41171b6b3c32, server=10641b2c0a69,39013,1730976230620 because future has completed 2024-11-07T10:43:53,756 INFO [StoreOpener-69160268ea309aa55ece0cd8528b941d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 69160268ea309aa55ece0cd8528b941d columnFamilyName colfamily11 2024-11-07T10:43:53,756 DEBUG [StoreOpener-69160268ea309aa55ece0cd8528b941d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,756 INFO [StoreOpener-72bf61b738a2f9b35913a192fc32815b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72bf61b738a2f9b35913a192fc32815b columnFamilyName colfamily31 2024-11-07T10:43:53,756 DEBUG [StoreOpener-72bf61b738a2f9b35913a192fc32815b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,756 INFO [StoreOpener-69160268ea309aa55ece0cd8528b941d-1 {}] regionserver.HStore(327): Store=69160268ea309aa55ece0cd8528b941d/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,757 INFO [StoreOpener-69160268ea309aa55ece0cd8528b941d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 69160268ea309aa55ece0cd8528b941d 2024-11-07T10:43:53,757 INFO [StoreOpener-72bf61b738a2f9b35913a192fc32815b-1 {}] regionserver.HStore(327): Store=72bf61b738a2f9b35913a192fc32815b/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,757 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 72bf61b738a2f9b35913a192fc32815b 2024-11-07T10:43:53,758 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72bf61b738a2f9b35913a192fc32815b 2024-11-07T10:43:53,759 INFO [StoreOpener-69160268ea309aa55ece0cd8528b941d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 69160268ea309aa55ece0cd8528b941d columnFamilyName colfamily21 2024-11-07T10:43:53,759 DEBUG [StoreOpener-69160268ea309aa55ece0cd8528b941d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,759 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72bf61b738a2f9b35913a192fc32815b 2024-11-07T10:43:53,760 INFO [StoreOpener-69160268ea309aa55ece0cd8528b941d-1 {}] regionserver.HStore(327): Store=69160268ea309aa55ece0cd8528b941d/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,760 INFO [StoreOpener-69160268ea309aa55ece0cd8528b941d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 69160268ea309aa55ece0cd8528b941d 2024-11-07T10:43:53,760 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 72bf61b738a2f9b35913a192fc32815b 2024-11-07T10:43:53,761 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 72bf61b738a2f9b35913a192fc32815b 2024-11-07T10:43:53,761 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:43:53,762 INFO [StoreOpener-69160268ea309aa55ece0cd8528b941d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 69160268ea309aa55ece0cd8528b941d columnFamilyName colfamily31 2024-11-07T10:43:53,762 DEBUG [StoreOpener-69160268ea309aa55ece0cd8528b941d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,762 INFO [StoreOpener-69160268ea309aa55ece0cd8528b941d-1 {}] regionserver.HStore(327): Store=69160268ea309aa55ece0cd8528b941d/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,762 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 69160268ea309aa55ece0cd8528b941d 2024-11-07T10:43:53,763 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/69160268ea309aa55ece0cd8528b941d 2024-11-07T10:43:53,763 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 72bf61b738a2f9b35913a192fc32815b 2024-11-07T10:43:53,764 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/69160268ea309aa55ece0cd8528b941d 2024-11-07T10:43:53,765 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cf36f48b84f67b77887428ec6ad5b31/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:43:53,766 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 2cf36f48b84f67b77887428ec6ad5b31; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63507401, jitterRate=-0.05366598069667816}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:43:53,766 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 69160268ea309aa55ece0cd8528b941d 2024-11-07T10:43:53,766 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2cf36f48b84f67b77887428ec6ad5b31 2024-11-07T10:43:53,766 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 69160268ea309aa55ece0cd8528b941d 2024-11-07T10:43:53,766 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 2cf36f48b84f67b77887428ec6ad5b31: Running coprocessor pre-open hook at 1730976233706Writing region info on filesystem at 1730976233706Initializing all the Stores at 1730976233708 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233708Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233724 (+16 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233724Cleaning up temporary data from old regions at 1730976233747 (+23 ms)Running coprocessor post-open hooks at 1730976233766 (+19 ms)Region opened successfully at 1730976233766 2024-11-07T10:43:53,767 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:43:53,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=7 2024-11-07T10:43:53,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure dbe69e720cc4f2a4a4bc41171b6b3c32, server=10641b2c0a69,39013,1730976230620 in 296 msec 2024-11-07T10:43:53,768 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1730976232769.2cf36f48b84f67b77887428ec6ad5b31., pid=46, masterSystemTime=1730976233503 2024-11-07T10:43:53,769 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 69160268ea309aa55ece0cd8528b941d 2024-11-07T10:43:53,771 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1730976232769.2cf36f48b84f67b77887428ec6ad5b31. 2024-11-07T10:43:53,771 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1730976232769.2cf36f48b84f67b77887428ec6ad5b31. 2024-11-07T10:43:53,771 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1730976232769.c71057666e459779c2fa5a4311f4e99c. 2024-11-07T10:43:53,772 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => c71057666e459779c2fa5a4311f4e99c, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1730976232769.c71057666e459779c2fa5a4311f4e99c.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-07T10:43:53,772 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true c71057666e459779c2fa5a4311f4e99c 2024-11-07T10:43:53,772 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1730976232769.c71057666e459779c2fa5a4311f4e99c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:53,772 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for c71057666e459779c2fa5a4311f4e99c 2024-11-07T10:43:53,772 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for c71057666e459779c2fa5a4311f4e99c 2024-11-07T10:43:53,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=dbe69e720cc4f2a4a4bc41171b6b3c32, ASSIGN in 630 msec 2024-11-07T10:43:53,774 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=2cf36f48b84f67b77887428ec6ad5b31, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,40573,1730976230476 2024-11-07T10:43:53,777 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72bf61b738a2f9b35913a192fc32815b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:43:53,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2cf36f48b84f67b77887428ec6ad5b31, server=10641b2c0a69,40573,1730976230476 because future has completed 2024-11-07T10:43:53,778 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/69160268ea309aa55ece0cd8528b941d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:43:53,778 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 72bf61b738a2f9b35913a192fc32815b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66500750, jitterRate=-0.009061604738235474}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:43:53,778 INFO [StoreOpener-c71057666e459779c2fa5a4311f4e99c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region c71057666e459779c2fa5a4311f4e99c 2024-11-07T10:43:53,778 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 72bf61b738a2f9b35913a192fc32815b 2024-11-07T10:43:53,778 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 72bf61b738a2f9b35913a192fc32815b: Running coprocessor pre-open hook at 1730976233710Writing region info on filesystem at 1730976233710Initializing all the Stores at 1730976233712 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233712Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233736 (+24 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233736Cleaning up temporary data from old regions at 1730976233761 (+25 ms)Running coprocessor post-open hooks at 1730976233778 (+17 ms)Region opened successfully at 1730976233778 2024-11-07T10:43:53,780 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 69160268ea309aa55ece0cd8528b941d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60681533, jitterRate=-0.09577469527721405}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:43:53,780 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 69160268ea309aa55ece0cd8528b941d 2024-11-07T10:43:53,780 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 69160268ea309aa55ece0cd8528b941d: Running coprocessor pre-open hook at 1730976233751Writing region info on filesystem at 1730976233751Initializing all the Stores at 1730976233752 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233752Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233753 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233753Cleaning up temporary data from old regions at 1730976233766 (+13 ms)Running coprocessor post-open hooks at 1730976233780 (+14 ms)Region opened successfully at 1730976233780 2024-11-07T10:43:53,780 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b., pid=37, masterSystemTime=1730976233474 2024-11-07T10:43:53,781 INFO [StoreOpener-c71057666e459779c2fa5a4311f4e99c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c71057666e459779c2fa5a4311f4e99c columnFamilyName colfamily11 2024-11-07T10:43:53,781 DEBUG [StoreOpener-c71057666e459779c2fa5a4311f4e99c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1dd5c615432f6d3976b7d3e591d0a96a, ASSIGN in 639 msec 2024-11-07T10:43:53,782 INFO [StoreOpener-c71057666e459779c2fa5a4311f4e99c-1 {}] regionserver.HStore(327): Store=c71057666e459779c2fa5a4311f4e99c/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,782 INFO [StoreOpener-c71057666e459779c2fa5a4311f4e99c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region c71057666e459779c2fa5a4311f4e99c 2024-11-07T10:43:53,782 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1730976232769.69160268ea309aa55ece0cd8528b941d., pid=48, masterSystemTime=1730976233582 2024-11-07T10:43:53,784 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b. 2024-11-07T10:43:53,784 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b. 2024-11-07T10:43:53,784 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71. 2024-11-07T10:43:53,784 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 8ecb70a5eda60ab5744089e2ffe59d71, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-07T10:43:53,784 INFO [StoreOpener-c71057666e459779c2fa5a4311f4e99c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c71057666e459779c2fa5a4311f4e99c columnFamilyName colfamily21 2024-11-07T10:43:53,784 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 8ecb70a5eda60ab5744089e2ffe59d71 2024-11-07T10:43:53,784 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:53,785 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 8ecb70a5eda60ab5744089e2ffe59d71 2024-11-07T10:43:53,785 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 8ecb70a5eda60ab5744089e2ffe59d71 2024-11-07T10:43:53,786 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1730976232769.69160268ea309aa55ece0cd8528b941d. 2024-11-07T10:43:53,786 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1730976232769.69160268ea309aa55ece0cd8528b941d. 2024-11-07T10:43:53,787 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1730976232769.f89575b73864cac8b7b164f417ef378f. 2024-11-07T10:43:53,787 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => f89575b73864cac8b7b164f417ef378f, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1730976232769.f89575b73864cac8b7b164f417ef378f.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-07T10:43:53,787 INFO [StoreOpener-8ecb70a5eda60ab5744089e2ffe59d71-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 8ecb70a5eda60ab5744089e2ffe59d71 2024-11-07T10:43:53,787 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true f89575b73864cac8b7b164f417ef378f 2024-11-07T10:43:53,788 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1730976232769.f89575b73864cac8b7b164f417ef378f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:53,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=24 2024-11-07T10:43:53,788 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for f89575b73864cac8b7b164f417ef378f 2024-11-07T10:43:53,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 2cf36f48b84f67b77887428ec6ad5b31, server=10641b2c0a69,40573,1730976230476 in 369 msec 2024-11-07T10:43:53,788 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for f89575b73864cac8b7b164f417ef378f 2024-11-07T10:43:53,788 DEBUG [StoreOpener-c71057666e459779c2fa5a4311f4e99c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,789 INFO [StoreOpener-c71057666e459779c2fa5a4311f4e99c-1 {}] regionserver.HStore(327): Store=c71057666e459779c2fa5a4311f4e99c/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,789 INFO [StoreOpener-c71057666e459779c2fa5a4311f4e99c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region c71057666e459779c2fa5a4311f4e99c 2024-11-07T10:43:53,789 INFO [StoreOpener-8ecb70a5eda60ab5744089e2ffe59d71-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ecb70a5eda60ab5744089e2ffe59d71 columnFamilyName colfamily11 2024-11-07T10:43:53,789 DEBUG [StoreOpener-8ecb70a5eda60ab5744089e2ffe59d71-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,789 INFO [StoreOpener-f89575b73864cac8b7b164f417ef378f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region f89575b73864cac8b7b164f417ef378f 2024-11-07T10:43:53,790 INFO [StoreOpener-8ecb70a5eda60ab5744089e2ffe59d71-1 {}] regionserver.HStore(327): Store=8ecb70a5eda60ab5744089e2ffe59d71/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,790 INFO [StoreOpener-8ecb70a5eda60ab5744089e2ffe59d71-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 8ecb70a5eda60ab5744089e2ffe59d71 2024-11-07T10:43:53,791 INFO [StoreOpener-c71057666e459779c2fa5a4311f4e99c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c71057666e459779c2fa5a4311f4e99c columnFamilyName colfamily31 2024-11-07T10:43:53,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2cf36f48b84f67b77887428ec6ad5b31, ASSIGN in 651 msec 2024-11-07T10:43:53,791 DEBUG [StoreOpener-c71057666e459779c2fa5a4311f4e99c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,792 INFO [StoreOpener-c71057666e459779c2fa5a4311f4e99c-1 {}] regionserver.HStore(327): Store=c71057666e459779c2fa5a4311f4e99c/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,792 INFO [StoreOpener-8ecb70a5eda60ab5744089e2ffe59d71-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ecb70a5eda60ab5744089e2ffe59d71 columnFamilyName colfamily21 2024-11-07T10:43:53,792 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for c71057666e459779c2fa5a4311f4e99c 2024-11-07T10:43:53,792 DEBUG [StoreOpener-8ecb70a5eda60ab5744089e2ffe59d71-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,792 INFO [StoreOpener-f89575b73864cac8b7b164f417ef378f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f89575b73864cac8b7b164f417ef378f columnFamilyName colfamily11 2024-11-07T10:43:53,792 DEBUG [StoreOpener-f89575b73864cac8b7b164f417ef378f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,793 INFO [StoreOpener-f89575b73864cac8b7b164f417ef378f-1 {}] regionserver.HStore(327): Store=f89575b73864cac8b7b164f417ef378f/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,793 INFO [StoreOpener-8ecb70a5eda60ab5744089e2ffe59d71-1 {}] regionserver.HStore(327): Store=8ecb70a5eda60ab5744089e2ffe59d71/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,793 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c71057666e459779c2fa5a4311f4e99c 2024-11-07T10:43:53,793 INFO [StoreOpener-f89575b73864cac8b7b164f417ef378f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region f89575b73864cac8b7b164f417ef378f 2024-11-07T10:43:53,793 INFO [StoreOpener-8ecb70a5eda60ab5744089e2ffe59d71-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 8ecb70a5eda60ab5744089e2ffe59d71 2024-11-07T10:43:53,794 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c71057666e459779c2fa5a4311f4e99c 2024-11-07T10:43:53,794 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=69160268ea309aa55ece0cd8528b941d, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,39013,1730976230620 2024-11-07T10:43:53,795 INFO [StoreOpener-f89575b73864cac8b7b164f417ef378f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f89575b73864cac8b7b164f417ef378f columnFamilyName colfamily21 2024-11-07T10:43:53,795 DEBUG [StoreOpener-f89575b73864cac8b7b164f417ef378f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,795 INFO [StoreOpener-8ecb70a5eda60ab5744089e2ffe59d71-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ecb70a5eda60ab5744089e2ffe59d71 columnFamilyName colfamily31 2024-11-07T10:43:53,795 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for c71057666e459779c2fa5a4311f4e99c 2024-11-07T10:43:53,795 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for c71057666e459779c2fa5a4311f4e99c 2024-11-07T10:43:53,796 DEBUG [StoreOpener-8ecb70a5eda60ab5744089e2ffe59d71-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,796 INFO [StoreOpener-f89575b73864cac8b7b164f417ef378f-1 {}] regionserver.HStore(327): Store=f89575b73864cac8b7b164f417ef378f/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,796 INFO [StoreOpener-f89575b73864cac8b7b164f417ef378f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region f89575b73864cac8b7b164f417ef378f 2024-11-07T10:43:53,796 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:43:53,797 INFO [StoreOpener-8ecb70a5eda60ab5744089e2ffe59d71-1 {}] regionserver.HStore(327): Store=8ecb70a5eda60ab5744089e2ffe59d71/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,797 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 8ecb70a5eda60ab5744089e2ffe59d71 2024-11-07T10:43:53,798 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ecb70a5eda60ab5744089e2ffe59d71 2024-11-07T10:43:53,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 69160268ea309aa55ece0cd8528b941d, server=10641b2c0a69,39013,1730976230620 because future has completed 2024-11-07T10:43:53,798 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ecb70a5eda60ab5744089e2ffe59d71 2024-11-07T10:43:53,799 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for c71057666e459779c2fa5a4311f4e99c 2024-11-07T10:43:53,799 INFO [StoreOpener-f89575b73864cac8b7b164f417ef378f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f89575b73864cac8b7b164f417ef378f columnFamilyName colfamily31 2024-11-07T10:43:53,799 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=72bf61b738a2f9b35913a192fc32815b, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,41401,1730976230578 2024-11-07T10:43:53,799 DEBUG [StoreOpener-f89575b73864cac8b7b164f417ef378f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,800 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 8ecb70a5eda60ab5744089e2ffe59d71 2024-11-07T10:43:53,800 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 8ecb70a5eda60ab5744089e2ffe59d71 2024-11-07T10:43:53,801 INFO [StoreOpener-f89575b73864cac8b7b164f417ef378f-1 {}] regionserver.HStore(327): Store=f89575b73864cac8b7b164f417ef378f/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,801 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for f89575b73864cac8b7b164f417ef378f 2024-11-07T10:43:53,802 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:43:53,803 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f89575b73864cac8b7b164f417ef378f 2024-11-07T10:43:53,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 72bf61b738a2f9b35913a192fc32815b, server=10641b2c0a69,41401,1730976230578 because future has completed 2024-11-07T10:43:53,804 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f89575b73864cac8b7b164f417ef378f 2024-11-07T10:43:53,804 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 8ecb70a5eda60ab5744089e2ffe59d71 2024-11-07T10:43:53,806 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for f89575b73864cac8b7b164f417ef378f 2024-11-07T10:43:53,806 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c71057666e459779c2fa5a4311f4e99c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:43:53,806 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for f89575b73864cac8b7b164f417ef378f 2024-11-07T10:43:53,807 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:43:53,807 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened c71057666e459779c2fa5a4311f4e99c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75145406, jitterRate=0.11975380778312683}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:43:53,807 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c71057666e459779c2fa5a4311f4e99c 2024-11-07T10:43:53,808 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for c71057666e459779c2fa5a4311f4e99c: Running coprocessor pre-open hook at 1730976233772Writing region info on filesystem at 1730976233772Initializing all the Stores at 1730976233774 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233774Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233778 (+4 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233778Cleaning up temporary data from old regions at 1730976233795 (+17 ms)Running coprocessor post-open hooks at 1730976233808 (+13 ms)Region opened successfully at 1730976233808 2024-11-07T10:43:53,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=25 2024-11-07T10:43:53,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 69160268ea309aa55ece0cd8528b941d, server=10641b2c0a69,39013,1730976230620 in 375 msec 2024-11-07T10:43:53,809 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ecb70a5eda60ab5744089e2ffe59d71/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:43:53,810 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1730976232769.c71057666e459779c2fa5a4311f4e99c., pid=34, masterSystemTime=1730976233503 2024-11-07T10:43:53,810 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 8ecb70a5eda60ab5744089e2ffe59d71; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61502629, jitterRate=-0.08353941142559052}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:43:53,810 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8ecb70a5eda60ab5744089e2ffe59d71 2024-11-07T10:43:53,810 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 8ecb70a5eda60ab5744089e2ffe59d71: Running coprocessor pre-open hook at 1730976233785Writing region info on filesystem at 1730976233785Initializing all the Stores at 1730976233786 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233787 (+1 ms)Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233787Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233787Cleaning up temporary data from old regions at 1730976233800 (+13 ms)Running coprocessor post-open hooks at 1730976233810 (+10 ms)Region opened successfully at 1730976233810 2024-11-07T10:43:53,811 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for f89575b73864cac8b7b164f417ef378f 2024-11-07T10:43:53,811 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71., pid=38, masterSystemTime=1730976233474 2024-11-07T10:43:53,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=69160268ea309aa55ece0cd8528b941d, ASSIGN in 672 msec 2024-11-07T10:43:53,814 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1730976232769.c71057666e459779c2fa5a4311f4e99c. 2024-11-07T10:43:53,814 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1730976232769.c71057666e459779c2fa5a4311f4e99c. 2024-11-07T10:43:53,815 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1730976232769.0a7d18696a90cfed5af1fbdb2a706b65. 2024-11-07T10:43:53,815 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f89575b73864cac8b7b164f417ef378f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:43:53,815 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 0a7d18696a90cfed5af1fbdb2a706b65, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1730976232769.0a7d18696a90cfed5af1fbdb2a706b65.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-07T10:43:53,815 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=c71057666e459779c2fa5a4311f4e99c, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,40573,1730976230476 2024-11-07T10:43:53,815 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 0a7d18696a90cfed5af1fbdb2a706b65 2024-11-07T10:43:53,815 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened f89575b73864cac8b7b164f417ef378f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74760873, jitterRate=0.11402381956577301}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:43:53,816 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1730976232769.0a7d18696a90cfed5af1fbdb2a706b65.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:53,816 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f89575b73864cac8b7b164f417ef378f 2024-11-07T10:43:53,816 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 0a7d18696a90cfed5af1fbdb2a706b65 2024-11-07T10:43:53,816 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for f89575b73864cac8b7b164f417ef378f: Running coprocessor pre-open hook at 1730976233788Writing region info on filesystem at 1730976233788Initializing all the Stores at 1730976233789 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233789Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233789Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233789Cleaning up temporary data from old regions at 1730976233806 (+17 ms)Running coprocessor post-open hooks at 1730976233816 (+10 ms)Region opened successfully at 1730976233816 2024-11-07T10:43:53,816 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 0a7d18696a90cfed5af1fbdb2a706b65 2024-11-07T10:43:53,816 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71. 2024-11-07T10:43:53,816 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71. 2024-11-07T10:43:53,816 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f. 2024-11-07T10:43:53,816 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 1701fae2bf725fdd94c16d1c6b8bc75f, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-07T10:43:53,817 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 1701fae2bf725fdd94c16d1c6b8bc75f 2024-11-07T10:43:53,817 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:53,817 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 1701fae2bf725fdd94c16d1c6b8bc75f 2024-11-07T10:43:53,817 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 1701fae2bf725fdd94c16d1c6b8bc75f 2024-11-07T10:43:53,818 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1730976232769.f89575b73864cac8b7b164f417ef378f., pid=56, masterSystemTime=1730976233582 2024-11-07T10:43:53,818 INFO [StoreOpener-0a7d18696a90cfed5af1fbdb2a706b65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 0a7d18696a90cfed5af1fbdb2a706b65 2024-11-07T10:43:53,819 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure c71057666e459779c2fa5a4311f4e99c, server=10641b2c0a69,40573,1730976230476 because future has completed 2024-11-07T10:43:53,820 INFO [StoreOpener-0a7d18696a90cfed5af1fbdb2a706b65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0a7d18696a90cfed5af1fbdb2a706b65 columnFamilyName colfamily11 2024-11-07T10:43:53,820 DEBUG [StoreOpener-0a7d18696a90cfed5af1fbdb2a706b65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,821 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1730976232769.f89575b73864cac8b7b164f417ef378f. 2024-11-07T10:43:53,821 INFO [StoreOpener-0a7d18696a90cfed5af1fbdb2a706b65-1 {}] regionserver.HStore(327): Store=0a7d18696a90cfed5af1fbdb2a706b65/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,821 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=8ecb70a5eda60ab5744089e2ffe59d71, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,41401,1730976230578 2024-11-07T10:43:53,821 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1730976232769.f89575b73864cac8b7b164f417ef378f. 2024-11-07T10:43:53,821 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1730976232769.0b29f5d988381aa77506fa3b92944854. 2024-11-07T10:43:53,821 INFO [StoreOpener-0a7d18696a90cfed5af1fbdb2a706b65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 0a7d18696a90cfed5af1fbdb2a706b65 2024-11-07T10:43:53,822 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 0b29f5d988381aa77506fa3b92944854, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1730976232769.0b29f5d988381aa77506fa3b92944854.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-07T10:43:53,822 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 0b29f5d988381aa77506fa3b92944854 2024-11-07T10:43:53,822 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1730976232769.0b29f5d988381aa77506fa3b92944854.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:53,822 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 0b29f5d988381aa77506fa3b92944854 2024-11-07T10:43:53,822 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 0b29f5d988381aa77506fa3b92944854 2024-11-07T10:43:53,824 INFO [StoreOpener-0a7d18696a90cfed5af1fbdb2a706b65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0a7d18696a90cfed5af1fbdb2a706b65 columnFamilyName colfamily21 2024-11-07T10:43:53,824 DEBUG [StoreOpener-0a7d18696a90cfed5af1fbdb2a706b65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,824 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ecb70a5eda60ab5744089e2ffe59d71, server=10641b2c0a69,41401,1730976230578 because future has completed 2024-11-07T10:43:53,825 INFO [StoreOpener-0b29f5d988381aa77506fa3b92944854-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 0b29f5d988381aa77506fa3b92944854 2024-11-07T10:43:53,825 INFO [StoreOpener-0a7d18696a90cfed5af1fbdb2a706b65-1 {}] regionserver.HStore(327): Store=0a7d18696a90cfed5af1fbdb2a706b65/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,825 INFO [StoreOpener-1701fae2bf725fdd94c16d1c6b8bc75f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 1701fae2bf725fdd94c16d1c6b8bc75f 2024-11-07T10:43:53,825 INFO [StoreOpener-0a7d18696a90cfed5af1fbdb2a706b65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 0a7d18696a90cfed5af1fbdb2a706b65 2024-11-07T10:43:53,827 INFO [StoreOpener-0b29f5d988381aa77506fa3b92944854-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b29f5d988381aa77506fa3b92944854 columnFamilyName colfamily11 2024-11-07T10:43:53,827 DEBUG [StoreOpener-0b29f5d988381aa77506fa3b92944854-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,828 INFO [StoreOpener-0a7d18696a90cfed5af1fbdb2a706b65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0a7d18696a90cfed5af1fbdb2a706b65 columnFamilyName colfamily31 2024-11-07T10:43:53,828 DEBUG [StoreOpener-0a7d18696a90cfed5af1fbdb2a706b65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,827 INFO [StoreOpener-1701fae2bf725fdd94c16d1c6b8bc75f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1701fae2bf725fdd94c16d1c6b8bc75f columnFamilyName colfamily11 2024-11-07T10:43:53,828 INFO [StoreOpener-0a7d18696a90cfed5af1fbdb2a706b65-1 {}] regionserver.HStore(327): Store=0a7d18696a90cfed5af1fbdb2a706b65/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,828 DEBUG [StoreOpener-1701fae2bf725fdd94c16d1c6b8bc75f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,829 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 0a7d18696a90cfed5af1fbdb2a706b65 2024-11-07T10:43:53,829 INFO [StoreOpener-0b29f5d988381aa77506fa3b92944854-1 {}] regionserver.HStore(327): Store=0b29f5d988381aa77506fa3b92944854/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,829 INFO [StoreOpener-0b29f5d988381aa77506fa3b92944854-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 0b29f5d988381aa77506fa3b92944854 2024-11-07T10:43:53,829 INFO [StoreOpener-1701fae2bf725fdd94c16d1c6b8bc75f-1 {}] regionserver.HStore(327): Store=1701fae2bf725fdd94c16d1c6b8bc75f/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,829 INFO [StoreOpener-1701fae2bf725fdd94c16d1c6b8bc75f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 1701fae2bf725fdd94c16d1c6b8bc75f 2024-11-07T10:43:53,830 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0a7d18696a90cfed5af1fbdb2a706b65 2024-11-07T10:43:53,830 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0a7d18696a90cfed5af1fbdb2a706b65 2024-11-07T10:43:53,830 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=17 2024-11-07T10:43:53,830 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 8ecb70a5eda60ab5744089e2ffe59d71, server=10641b2c0a69,41401,1730976230578 in 456 msec 2024-11-07T10:43:53,831 INFO [StoreOpener-0b29f5d988381aa77506fa3b92944854-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b29f5d988381aa77506fa3b92944854 columnFamilyName colfamily21 2024-11-07T10:43:53,831 DEBUG [StoreOpener-0b29f5d988381aa77506fa3b92944854-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,831 INFO [StoreOpener-1701fae2bf725fdd94c16d1c6b8bc75f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1701fae2bf725fdd94c16d1c6b8bc75f columnFamilyName colfamily21 2024-11-07T10:43:53,832 DEBUG [StoreOpener-1701fae2bf725fdd94c16d1c6b8bc75f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,832 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 0a7d18696a90cfed5af1fbdb2a706b65 2024-11-07T10:43:53,832 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 0a7d18696a90cfed5af1fbdb2a706b65 2024-11-07T10:43:53,832 INFO [StoreOpener-0b29f5d988381aa77506fa3b92944854-1 {}] regionserver.HStore(327): Store=0b29f5d988381aa77506fa3b92944854/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,832 INFO [StoreOpener-0b29f5d988381aa77506fa3b92944854-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 0b29f5d988381aa77506fa3b92944854 2024-11-07T10:43:53,832 INFO [StoreOpener-1701fae2bf725fdd94c16d1c6b8bc75f-1 {}] regionserver.HStore(327): Store=1701fae2bf725fdd94c16d1c6b8bc75f/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,833 INFO [StoreOpener-1701fae2bf725fdd94c16d1c6b8bc75f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 1701fae2bf725fdd94c16d1c6b8bc75f 2024-11-07T10:43:53,833 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:43:53,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8ecb70a5eda60ab5744089e2ffe59d71, ASSIGN in 694 msec 2024-11-07T10:43:53,834 INFO [StoreOpener-0b29f5d988381aa77506fa3b92944854-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b29f5d988381aa77506fa3b92944854 columnFamilyName colfamily31 2024-11-07T10:43:53,834 DEBUG [StoreOpener-0b29f5d988381aa77506fa3b92944854-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,834 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 0a7d18696a90cfed5af1fbdb2a706b65 2024-11-07T10:43:53,835 INFO [StoreOpener-1701fae2bf725fdd94c16d1c6b8bc75f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1701fae2bf725fdd94c16d1c6b8bc75f columnFamilyName colfamily31 2024-11-07T10:43:53,835 INFO [StoreOpener-0b29f5d988381aa77506fa3b92944854-1 {}] regionserver.HStore(327): Store=0b29f5d988381aa77506fa3b92944854/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,835 DEBUG [StoreOpener-1701fae2bf725fdd94c16d1c6b8bc75f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,835 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 0b29f5d988381aa77506fa3b92944854 2024-11-07T10:43:53,835 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=f89575b73864cac8b7b164f417ef378f, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,39013,1730976230620 2024-11-07T10:43:53,835 INFO [StoreOpener-1701fae2bf725fdd94c16d1c6b8bc75f-1 {}] regionserver.HStore(327): Store=1701fae2bf725fdd94c16d1c6b8bc75f/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,836 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 1701fae2bf725fdd94c16d1c6b8bc75f 2024-11-07T10:43:53,836 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0b29f5d988381aa77506fa3b92944854 2024-11-07T10:43:53,838 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1701fae2bf725fdd94c16d1c6b8bc75f 2024-11-07T10:43:53,838 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0b29f5d988381aa77506fa3b92944854 2024-11-07T10:43:53,838 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1701fae2bf725fdd94c16d1c6b8bc75f 2024-11-07T10:43:53,839 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure f89575b73864cac8b7b164f417ef378f, server=10641b2c0a69,39013,1730976230620 because future has completed 2024-11-07T10:43:53,839 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0a7d18696a90cfed5af1fbdb2a706b65/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:43:53,840 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 0b29f5d988381aa77506fa3b92944854 2024-11-07T10:43:53,840 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 0b29f5d988381aa77506fa3b92944854 2024-11-07T10:43:53,840 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 0a7d18696a90cfed5af1fbdb2a706b65; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72036877, jitterRate=0.07343311607837677}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:43:53,840 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0a7d18696a90cfed5af1fbdb2a706b65 2024-11-07T10:43:53,840 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 0a7d18696a90cfed5af1fbdb2a706b65: Running coprocessor pre-open hook at 1730976233816Writing region info on filesystem at 1730976233816Initializing all the Stores at 1730976233818 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233818Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233818Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233818Cleaning up temporary data from old regions at 1730976233832 (+14 ms)Running coprocessor post-open hooks at 1730976233840 (+8 ms)Region opened successfully at 1730976233840 2024-11-07T10:43:53,841 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:43:53,842 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 1701fae2bf725fdd94c16d1c6b8bc75f 2024-11-07T10:43:53,842 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 1701fae2bf725fdd94c16d1c6b8bc75f 2024-11-07T10:43:53,842 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1730976232769.0a7d18696a90cfed5af1fbdb2a706b65., pid=42, masterSystemTime=1730976233503 2024-11-07T10:43:53,842 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:43:53,844 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 0b29f5d988381aa77506fa3b92944854 2024-11-07T10:43:53,845 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 1701fae2bf725fdd94c16d1c6b8bc75f 2024-11-07T10:43:53,846 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=30 2024-11-07T10:43:53,846 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure c71057666e459779c2fa5a4311f4e99c, server=10641b2c0a69,40573,1730976230476 in 492 msec 2024-11-07T10:43:53,847 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=0a7d18696a90cfed5af1fbdb2a706b65, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,40573,1730976230476 2024-11-07T10:43:53,847 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43605 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=10641b2c0a69,39013,1730976230620, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f89575b73864cac8b7b164f417ef378f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-07T10:43:53,849 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1730976232769.0a7d18696a90cfed5af1fbdb2a706b65. 2024-11-07T10:43:53,849 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1730976232769.0a7d18696a90cfed5af1fbdb2a706b65. 2024-11-07T10:43:53,850 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1730976232769.bcc3491e6200809e4c9c7d529da2c4cf. 2024-11-07T10:43:53,850 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => bcc3491e6200809e4c9c7d529da2c4cf, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1730976232769.bcc3491e6200809e4c9c7d529da2c4cf.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-07T10:43:53,850 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true bcc3491e6200809e4c9c7d529da2c4cf 2024-11-07T10:43:53,850 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1730976232769.bcc3491e6200809e4c9c7d529da2c4cf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:53,850 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for bcc3491e6200809e4c9c7d529da2c4cf 2024-11-07T10:43:53,850 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for bcc3491e6200809e4c9c7d529da2c4cf 2024-11-07T10:43:53,851 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0a7d18696a90cfed5af1fbdb2a706b65, server=10641b2c0a69,40573,1730976230476 because future has completed 2024-11-07T10:43:53,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c71057666e459779c2fa5a4311f4e99c, ASSIGN in 713 msec 2024-11-07T10:43:53,853 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1701fae2bf725fdd94c16d1c6b8bc75f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:43:53,853 INFO [StoreOpener-bcc3491e6200809e4c9c7d529da2c4cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region bcc3491e6200809e4c9c7d529da2c4cf 2024-11-07T10:43:53,854 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 1701fae2bf725fdd94c16d1c6b8bc75f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68535462, jitterRate=0.02125796675682068}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:43:53,854 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1701fae2bf725fdd94c16d1c6b8bc75f 2024-11-07T10:43:53,854 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 1701fae2bf725fdd94c16d1c6b8bc75f: Running coprocessor pre-open hook at 1730976233817Writing region info on filesystem at 1730976233817Initializing all the Stores at 1730976233819 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233819Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233824 (+5 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233824Cleaning up temporary data from old regions at 1730976233842 (+18 ms)Running coprocessor post-open hooks at 1730976233854 (+12 ms)Region opened successfully at 1730976233854 2024-11-07T10:43:53,855 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0b29f5d988381aa77506fa3b92944854/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:43:53,856 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 0b29f5d988381aa77506fa3b92944854; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65165382, jitterRate=-0.02896013855934143}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:43:53,856 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0b29f5d988381aa77506fa3b92944854 2024-11-07T10:43:53,856 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 0b29f5d988381aa77506fa3b92944854: Running coprocessor pre-open hook at 1730976233822Writing region info on filesystem at 1730976233822Initializing all the Stores at 1730976233824 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233824Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233825 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233825Cleaning up temporary data from old regions at 1730976233840 (+15 ms)Running coprocessor post-open hooks at 1730976233856 (+16 ms)Region opened successfully at 1730976233856 2024-11-07T10:43:53,856 INFO [StoreOpener-bcc3491e6200809e4c9c7d529da2c4cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bcc3491e6200809e4c9c7d529da2c4cf columnFamilyName colfamily11 2024-11-07T10:43:53,856 DEBUG [StoreOpener-bcc3491e6200809e4c9c7d529da2c4cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,857 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f., pid=39, masterSystemTime=1730976233474 2024-11-07T10:43:53,857 INFO [StoreOpener-bcc3491e6200809e4c9c7d529da2c4cf-1 {}] regionserver.HStore(327): Store=bcc3491e6200809e4c9c7d529da2c4cf/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,857 INFO [StoreOpener-bcc3491e6200809e4c9c7d529da2c4cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region bcc3491e6200809e4c9c7d529da2c4cf 2024-11-07T10:43:53,857 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1730976232769.0b29f5d988381aa77506fa3b92944854., pid=51, masterSystemTime=1730976233582 2024-11-07T10:43:53,860 INFO [StoreOpener-bcc3491e6200809e4c9c7d529da2c4cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bcc3491e6200809e4c9c7d529da2c4cf columnFamilyName colfamily21 2024-11-07T10:43:53,860 DEBUG [StoreOpener-bcc3491e6200809e4c9c7d529da2c4cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,860 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f. 2024-11-07T10:43:53,860 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f. 2024-11-07T10:43:53,861 INFO [StoreOpener-bcc3491e6200809e4c9c7d529da2c4cf-1 {}] regionserver.HStore(327): Store=bcc3491e6200809e4c9c7d529da2c4cf/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=15 2024-11-07T10:43:53,861 INFO [StoreOpener-bcc3491e6200809e4c9c7d529da2c4cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region bcc3491e6200809e4c9c7d529da2c4cf 2024-11-07T10:43:53,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 0a7d18696a90cfed5af1fbdb2a706b65, server=10641b2c0a69,40573,1730976230476 in 458 msec 2024-11-07T10:43:53,861 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1730976232769.0b29f5d988381aa77506fa3b92944854. 2024-11-07T10:43:53,861 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1730976232769.0b29f5d988381aa77506fa3b92944854. 2024-11-07T10:43:53,862 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1730976232769.1baabb362104b46ecb78fe236391ca6f. 2024-11-07T10:43:53,862 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 1baabb362104b46ecb78fe236391ca6f, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1730976232769.1baabb362104b46ecb78fe236391ca6f.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-07T10:43:53,862 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 1baabb362104b46ecb78fe236391ca6f 2024-11-07T10:43:53,862 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1730976232769.1baabb362104b46ecb78fe236391ca6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:53,863 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 1baabb362104b46ecb78fe236391ca6f 2024-11-07T10:43:53,863 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 1baabb362104b46ecb78fe236391ca6f 2024-11-07T10:43:53,864 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=0b29f5d988381aa77506fa3b92944854, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,39013,1730976230620 2024-11-07T10:43:53,864 INFO [StoreOpener-bcc3491e6200809e4c9c7d529da2c4cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bcc3491e6200809e4c9c7d529da2c4cf columnFamilyName colfamily31 2024-11-07T10:43:53,864 DEBUG [StoreOpener-bcc3491e6200809e4c9c7d529da2c4cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,865 INFO [StoreOpener-bcc3491e6200809e4c9c7d529da2c4cf-1 {}] regionserver.HStore(327): Store=bcc3491e6200809e4c9c7d529da2c4cf/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,865 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for bcc3491e6200809e4c9c7d529da2c4cf 2024-11-07T10:43:53,866 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcc3491e6200809e4c9c7d529da2c4cf 2024-11-07T10:43:53,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b29f5d988381aa77506fa3b92944854, server=10641b2c0a69,39013,1730976230620 because future has completed 2024-11-07T10:43:53,867 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcc3491e6200809e4c9c7d529da2c4cf 2024-11-07T10:43:53,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0a7d18696a90cfed5af1fbdb2a706b65, ASSIGN in 729 msec 2024-11-07T10:43:53,869 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for bcc3491e6200809e4c9c7d529da2c4cf 2024-11-07T10:43:53,869 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for bcc3491e6200809e4c9c7d529da2c4cf 2024-11-07T10:43:53,870 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:43:53,872 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for bcc3491e6200809e4c9c7d529da2c4cf 2024-11-07T10:43:53,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=28 2024-11-07T10:43:53,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 0b29f5d988381aa77506fa3b92944854, server=10641b2c0a69,39013,1730976230620 in 427 msec 2024-11-07T10:43:53,874 INFO [StoreOpener-1baabb362104b46ecb78fe236391ca6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 1baabb362104b46ecb78fe236391ca6f 2024-11-07T10:43:53,875 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcc3491e6200809e4c9c7d529da2c4cf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:43:53,876 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened bcc3491e6200809e4c9c7d529da2c4cf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64625287, jitterRate=-0.03700818121433258}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:43:53,876 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bcc3491e6200809e4c9c7d529da2c4cf 2024-11-07T10:43:53,876 INFO [StoreOpener-1baabb362104b46ecb78fe236391ca6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1baabb362104b46ecb78fe236391ca6f columnFamilyName colfamily11 2024-11-07T10:43:53,876 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for bcc3491e6200809e4c9c7d529da2c4cf: Running coprocessor pre-open hook at 1730976233851Writing region info on filesystem at 1730976233851Initializing all the Stores at 1730976233852 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233852Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233853 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233853Cleaning up temporary data from old regions at 1730976233869 (+16 ms)Running coprocessor post-open hooks at 1730976233876 (+7 ms)Region opened successfully at 1730976233876 2024-11-07T10:43:53,876 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0b29f5d988381aa77506fa3b92944854, ASSIGN in 737 msec 2024-11-07T10:43:53,876 DEBUG [StoreOpener-1baabb362104b46ecb78fe236391ca6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,877 INFO [StoreOpener-1baabb362104b46ecb78fe236391ca6f-1 {}] regionserver.HStore(327): Store=1baabb362104b46ecb78fe236391ca6f/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,878 INFO [StoreOpener-1baabb362104b46ecb78fe236391ca6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 1baabb362104b46ecb78fe236391ca6f 2024-11-07T10:43:53,879 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=1701fae2bf725fdd94c16d1c6b8bc75f, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,41401,1730976230578 2024-11-07T10:43:53,879 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43605 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=10641b2c0a69,41401,1730976230578, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=72bf61b738a2f9b35913a192fc32815b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-07T10:43:53,879 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43605 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=10641b2c0a69,41401,1730976230578, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1701fae2bf725fdd94c16d1c6b8bc75f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-07T10:43:53,879 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1730976232769.bcc3491e6200809e4c9c7d529da2c4cf., pid=52, masterSystemTime=1730976233503 2024-11-07T10:43:53,881 INFO [StoreOpener-1baabb362104b46ecb78fe236391ca6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1baabb362104b46ecb78fe236391ca6f columnFamilyName colfamily21 2024-11-07T10:43:53,881 DEBUG [StoreOpener-1baabb362104b46ecb78fe236391ca6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,882 INFO [StoreOpener-1baabb362104b46ecb78fe236391ca6f-1 {}] regionserver.HStore(327): Store=1baabb362104b46ecb78fe236391ca6f/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,882 INFO [StoreOpener-1baabb362104b46ecb78fe236391ca6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 1baabb362104b46ecb78fe236391ca6f 2024-11-07T10:43:53,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1701fae2bf725fdd94c16d1c6b8bc75f, server=10641b2c0a69,41401,1730976230578 because future has completed 2024-11-07T10:43:53,883 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1730976232769.bcc3491e6200809e4c9c7d529da2c4cf. 2024-11-07T10:43:53,883 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1730976232769.bcc3491e6200809e4c9c7d529da2c4cf. 2024-11-07T10:43:53,884 INFO [StoreOpener-1baabb362104b46ecb78fe236391ca6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1baabb362104b46ecb78fe236391ca6f columnFamilyName colfamily31 2024-11-07T10:43:53,884 DEBUG [StoreOpener-1baabb362104b46ecb78fe236391ca6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,885 INFO [StoreOpener-1baabb362104b46ecb78fe236391ca6f-1 {}] regionserver.HStore(327): Store=1baabb362104b46ecb78fe236391ca6f/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,885 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 1baabb362104b46ecb78fe236391ca6f 2024-11-07T10:43:53,886 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1baabb362104b46ecb78fe236391ca6f 2024-11-07T10:43:53,886 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1baabb362104b46ecb78fe236391ca6f 2024-11-07T10:43:53,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=13 2024-11-07T10:43:53,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure f89575b73864cac8b7b164f417ef378f, server=10641b2c0a69,39013,1730976230620 in 417 msec 2024-11-07T10:43:53,888 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 1baabb362104b46ecb78fe236391ca6f 2024-11-07T10:43:53,888 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 1baabb362104b46ecb78fe236391ca6f 2024-11-07T10:43:53,888 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:43:53,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f89575b73864cac8b7b164f417ef378f, ASSIGN in 750 msec 2024-11-07T10:43:53,890 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 1baabb362104b46ecb78fe236391ca6f 2024-11-07T10:43:53,890 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=bcc3491e6200809e4c9c7d529da2c4cf, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,40573,1730976230476 2024-11-07T10:43:53,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=23 2024-11-07T10:43:53,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 1701fae2bf725fdd94c16d1c6b8bc75f, server=10641b2c0a69,41401,1730976230578 in 518 msec 2024-11-07T10:43:53,899 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1701fae2bf725fdd94c16d1c6b8bc75f, ASSIGN in 760 msec 2024-11-07T10:43:53,901 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure bcc3491e6200809e4c9c7d529da2c4cf, server=10641b2c0a69,40573,1730976230476 because future has completed 2024-11-07T10:43:53,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=14 2024-11-07T10:43:53,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 72bf61b738a2f9b35913a192fc32815b, server=10641b2c0a69,41401,1730976230578 in 533 msec 2024-11-07T10:43:53,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=72bf61b738a2f9b35913a192fc32815b, ASSIGN in 766 msec 2024-11-07T10:43:53,906 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1baabb362104b46ecb78fe236391ca6f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:43:53,907 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 1baabb362104b46ecb78fe236391ca6f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74884786, jitterRate=0.11587026715278625}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:43:53,907 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1baabb362104b46ecb78fe236391ca6f 2024-11-07T10:43:53,907 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 1baabb362104b46ecb78fe236391ca6f: Running coprocessor pre-open hook at 1730976233863Writing region info on filesystem at 1730976233863Initializing all the Stores at 1730976233864 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233864Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233874 (+10 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233874Cleaning up temporary data from old regions at 1730976233888 (+14 ms)Running coprocessor post-open hooks at 1730976233907 (+19 ms)Region opened successfully at 1730976233907 2024-11-07T10:43:53,908 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1730976232769.1baabb362104b46ecb78fe236391ca6f., pid=53, masterSystemTime=1730976233582 2024-11-07T10:43:53,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=6 2024-11-07T10:43:53,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure bcc3491e6200809e4c9c7d529da2c4cf, server=10641b2c0a69,40573,1730976230476 in 460 msec 2024-11-07T10:43:53,913 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1730976232769.1baabb362104b46ecb78fe236391ca6f. 2024-11-07T10:43:53,913 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1730976232769.1baabb362104b46ecb78fe236391ca6f. 2024-11-07T10:43:53,913 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d. 2024-11-07T10:43:53,913 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 3f6021bbda5b8b7b00318a57f02cd77d, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-07T10:43:53,914 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bcc3491e6200809e4c9c7d529da2c4cf, ASSIGN in 775 msec 2024-11-07T10:43:53,914 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 3f6021bbda5b8b7b00318a57f02cd77d 2024-11-07T10:43:53,914 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:43:53,914 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 3f6021bbda5b8b7b00318a57f02cd77d 2024-11-07T10:43:53,914 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 3f6021bbda5b8b7b00318a57f02cd77d 2024-11-07T10:43:53,915 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1baabb362104b46ecb78fe236391ca6f, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,39013,1730976230620 2024-11-07T10:43:53,919 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1baabb362104b46ecb78fe236391ca6f, server=10641b2c0a69,39013,1730976230620 because future has completed 2024-11-07T10:43:53,922 INFO [StoreOpener-3f6021bbda5b8b7b00318a57f02cd77d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 3f6021bbda5b8b7b00318a57f02cd77d 2024-11-07T10:43:53,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=16 2024-11-07T10:43:53,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1baabb362104b46ecb78fe236391ca6f, server=10641b2c0a69,39013,1730976230620 in 467 msec 2024-11-07T10:43:53,924 INFO [StoreOpener-3f6021bbda5b8b7b00318a57f02cd77d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f6021bbda5b8b7b00318a57f02cd77d columnFamilyName colfamily11 2024-11-07T10:43:53,924 DEBUG [StoreOpener-3f6021bbda5b8b7b00318a57f02cd77d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,925 INFO [StoreOpener-3f6021bbda5b8b7b00318a57f02cd77d-1 {}] regionserver.HStore(327): Store=3f6021bbda5b8b7b00318a57f02cd77d/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,925 INFO [StoreOpener-3f6021bbda5b8b7b00318a57f02cd77d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 3f6021bbda5b8b7b00318a57f02cd77d 2024-11-07T10:43:53,925 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1baabb362104b46ecb78fe236391ca6f, ASSIGN in 786 msec 2024-11-07T10:43:53,926 INFO [StoreOpener-3f6021bbda5b8b7b00318a57f02cd77d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f6021bbda5b8b7b00318a57f02cd77d columnFamilyName colfamily21 2024-11-07T10:43:53,926 DEBUG [StoreOpener-3f6021bbda5b8b7b00318a57f02cd77d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,927 INFO [StoreOpener-3f6021bbda5b8b7b00318a57f02cd77d-1 {}] regionserver.HStore(327): Store=3f6021bbda5b8b7b00318a57f02cd77d/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,927 INFO [StoreOpener-3f6021bbda5b8b7b00318a57f02cd77d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 3f6021bbda5b8b7b00318a57f02cd77d 2024-11-07T10:43:53,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T10:43:53,929 INFO [StoreOpener-3f6021bbda5b8b7b00318a57f02cd77d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3f6021bbda5b8b7b00318a57f02cd77d columnFamilyName colfamily31 2024-11-07T10:43:53,929 DEBUG [StoreOpener-3f6021bbda5b8b7b00318a57f02cd77d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:43:53,929 INFO [StoreOpener-3f6021bbda5b8b7b00318a57f02cd77d-1 {}] regionserver.HStore(327): Store=3f6021bbda5b8b7b00318a57f02cd77d/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:43:53,930 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 3f6021bbda5b8b7b00318a57f02cd77d 2024-11-07T10:43:53,930 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3f6021bbda5b8b7b00318a57f02cd77d 2024-11-07T10:43:53,931 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3f6021bbda5b8b7b00318a57f02cd77d 2024-11-07T10:43:53,932 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 3f6021bbda5b8b7b00318a57f02cd77d 2024-11-07T10:43:53,932 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 3f6021bbda5b8b7b00318a57f02cd77d 2024-11-07T10:43:53,933 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:43:53,934 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 3f6021bbda5b8b7b00318a57f02cd77d 2024-11-07T10:43:53,937 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3f6021bbda5b8b7b00318a57f02cd77d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-07T10:43:53,937 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 3f6021bbda5b8b7b00318a57f02cd77d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75186160, jitterRate=0.1203610897064209}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:43:53,937 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3f6021bbda5b8b7b00318a57f02cd77d 2024-11-07T10:43:53,937 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 3f6021bbda5b8b7b00318a57f02cd77d: Running coprocessor pre-open hook at 1730976233915Writing region info on filesystem at 1730976233915Initializing all the Stores at 1730976233916 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233916Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233922 (+6 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976233922Cleaning up temporary data from old regions at 1730976233932 (+10 ms)Running coprocessor post-open hooks at 1730976233937 (+5 ms)Region opened successfully at 1730976233937 2024-11-07T10:43:53,938 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d., pid=54, masterSystemTime=1730976233582 2024-11-07T10:43:53,941 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d. 2024-11-07T10:43:53,941 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d. 2024-11-07T10:43:53,942 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=3f6021bbda5b8b7b00318a57f02cd77d, regionState=OPEN, openSeqNum=2, regionLocation=10641b2c0a69,39013,1730976230620 2024-11-07T10:43:53,945 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3f6021bbda5b8b7b00318a57f02cd77d, server=10641b2c0a69,39013,1730976230620 because future has completed 2024-11-07T10:43:53,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=9 2024-11-07T10:43:53,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 3f6021bbda5b8b7b00318a57f02cd77d, server=10641b2c0a69,39013,1730976230620 in 488 msec 2024-11-07T10:43:53,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=4 2024-11-07T10:43:53,956 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3f6021bbda5b8b7b00318a57f02cd77d, ASSIGN in 814 msec 2024-11-07T10:43:53,957 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-07T10:43:53,957 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1730976233957"}]},"ts":"1730976233957"} 2024-11-07T10:43:53,960 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, state=ENABLED in hbase:meta 2024-11-07T10:43:53,961 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-07T10:43:53,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true in 1.1850 sec 2024-11-07T10:43:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43605 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-07T10:43:54,939 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true get assigned. Timeout = 60000ms 2024-11-07T10:43:54,939 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true completed 2024-11-07T10:43:54,940 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T10:43:54,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41401 {}] regionserver.StoreScanner(1138): Switch to stream read (scanned=32802 bytes) of info 2024-11-07T10:43:54,962 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true assigned to meta. Checking AM states. 2024-11-07T10:43:54,962 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T10:43:54,963 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true assigned. 2024-11-07T10:43:54,995 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aaa', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:54,997 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aab', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,001 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aad', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,002 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aac', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,003 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aaf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,004 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aae', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,005 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aah', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,006 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aag', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,055 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1730976232769.bcc3491e6200809e4c9c7d529da2c4cf., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,057 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1730976232769.bcc3491e6200809e4c9c7d529da2c4cf., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,058 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1730976232769.bcc3491e6200809e4c9c7d529da2c4cf., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,059 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbe', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1730976232769.bcc3491e6200809e4c9c7d529da2c4cf., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,061 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1730976232769.bcc3491e6200809e4c9c7d529da2c4cf., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,062 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1730976232769.bcc3491e6200809e4c9c7d529da2c4cf., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,098 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1730976232769.dbe69e720cc4f2a4a4bc41171b6b3c32., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,100 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1730976232769.dbe69e720cc4f2a4a4bc41171b6b3c32., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,101 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='cce', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1730976232769.dbe69e720cc4f2a4a4bc41171b6b3c32., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,104 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1730976232769.dbe69e720cc4f2a4a4bc41171b6b3c32., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,105 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='cch', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1730976232769.dbe69e720cc4f2a4a4bc41171b6b3c32., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,106 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1730976232769.dbe69e720cc4f2a4a4bc41171b6b3c32., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,141 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,143 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='dde', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,145 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,146 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,147 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,148 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,187 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eef', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,188 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eee', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,190 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eeg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,191 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eeh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,192 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eei', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,193 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eej', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,196 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eek', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,197 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eel', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,244 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='fff', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1730976232769.ddeeab6f2e44d85eb1ee916d3c5e8e9f., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,246 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1730976232769.ddeeab6f2e44d85eb1ee916d3c5e8e9f., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,248 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1730976232769.ddeeab6f2e44d85eb1ee916d3c5e8e9f., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,250 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1730976232769.ddeeab6f2e44d85eb1ee916d3c5e8e9f., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,252 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1730976232769.ddeeab6f2e44d85eb1ee916d3c5e8e9f., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,254 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1730976232769.ddeeab6f2e44d85eb1ee916d3c5e8e9f., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,256 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1730976232769.ddeeab6f2e44d85eb1ee916d3c5e8e9f., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,283 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,286 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,287 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,289 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,290 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,292 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,336 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1730976232769.c3ab41a5354d389521b694050824788e., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,339 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1730976232769.c3ab41a5354d389521b694050824788e., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,341 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1730976232769.c3ab41a5354d389521b694050824788e., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,342 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1730976232769.c3ab41a5354d389521b694050824788e., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,344 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1730976232769.c3ab41a5354d389521b694050824788e., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,346 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1730976232769.c3ab41a5354d389521b694050824788e., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,379 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iii', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1730976232769.f89575b73864cac8b7b164f417ef378f., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,381 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iij', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1730976232769.f89575b73864cac8b7b164f417ef378f., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,383 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iik', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1730976232769.f89575b73864cac8b7b164f417ef378f., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,385 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iil', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1730976232769.f89575b73864cac8b7b164f417ef378f., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,387 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iim', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1730976232769.f89575b73864cac8b7b164f417ef378f., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,389 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iin', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1730976232769.f89575b73864cac8b7b164f417ef378f., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,421 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,424 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,425 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,427 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,429 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,431 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,432 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,474 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1730976232769.0a7d18696a90cfed5af1fbdb2a706b65., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,477 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1730976232769.0a7d18696a90cfed5af1fbdb2a706b65., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,480 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1730976232769.0a7d18696a90cfed5af1fbdb2a706b65., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,482 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1730976232769.0a7d18696a90cfed5af1fbdb2a706b65., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,484 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kko', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1730976232769.0a7d18696a90cfed5af1fbdb2a706b65., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,486 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1730976232769.0a7d18696a90cfed5af1fbdb2a706b65., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,488 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1730976232769.0a7d18696a90cfed5af1fbdb2a706b65., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,518 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='lll', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1730976232769.1baabb362104b46ecb78fe236391ca6f., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,521 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1730976232769.1baabb362104b46ecb78fe236391ca6f., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,523 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='lln', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1730976232769.1baabb362104b46ecb78fe236391ca6f., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,525 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1730976232769.1baabb362104b46ecb78fe236391ca6f., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,527 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1730976232769.1baabb362104b46ecb78fe236391ca6f., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,529 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1730976232769.1baabb362104b46ecb78fe236391ca6f., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,564 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,566 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,568 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,570 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,572 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,574 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,607 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1730976232769.9b830813c6b6c3ebcdb59061af10faad., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,609 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nno', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1730976232769.9b830813c6b6c3ebcdb59061af10faad., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,611 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1730976232769.9b830813c6b6c3ebcdb59061af10faad., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,612 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1730976232769.9b830813c6b6c3ebcdb59061af10faad., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,614 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1730976232769.9b830813c6b6c3ebcdb59061af10faad., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,616 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nns', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1730976232769.9b830813c6b6c3ebcdb59061af10faad., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,618 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1730976232769.9b830813c6b6c3ebcdb59061af10faad., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,620 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1730976232769.9b830813c6b6c3ebcdb59061af10faad., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,650 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ooo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1730976232769.dec39ef6d94f3a519151705fce17adc4., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,652 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oop', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1730976232769.dec39ef6d94f3a519151705fce17adc4., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,654 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ooq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1730976232769.dec39ef6d94f3a519151705fce17adc4., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,656 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oor', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1730976232769.dec39ef6d94f3a519151705fce17adc4., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,658 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oot', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1730976232769.dec39ef6d94f3a519151705fce17adc4., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,659 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oos', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1730976232769.dec39ef6d94f3a519151705fce17adc4., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,693 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,695 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,697 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,699 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='pps', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,701 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,703 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,705 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,742 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1730976232769.64da674e3a9ec8e7458eb07874d9f204., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,745 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1730976232769.64da674e3a9ec8e7458eb07874d9f204., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,746 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqs', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1730976232769.64da674e3a9ec8e7458eb07874d9f204., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,748 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1730976232769.64da674e3a9ec8e7458eb07874d9f204., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,750 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1730976232769.64da674e3a9ec8e7458eb07874d9f204., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,782 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1730976232769.9fba8e9ebafee4e67bfa1295ac44d4a5., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,784 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrs', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1730976232769.9fba8e9ebafee4e67bfa1295ac44d4a5., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,786 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1730976232769.9fba8e9ebafee4e67bfa1295ac44d4a5., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,788 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rru', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1730976232769.9fba8e9ebafee4e67bfa1295ac44d4a5., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,789 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1730976232769.9fba8e9ebafee4e67bfa1295ac44d4a5., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,824 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='sss', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,826 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='sst', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,828 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,830 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,832 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,834 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,879 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1730976232769.2cf36f48b84f67b77887428ec6ad5b31., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,881 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1730976232769.2cf36f48b84f67b77887428ec6ad5b31., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,883 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1730976232769.2cf36f48b84f67b77887428ec6ad5b31., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,885 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1730976232769.2cf36f48b84f67b77887428ec6ad5b31., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,886 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1730976232769.2cf36f48b84f67b77887428ec6ad5b31., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:55,921 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1730976232769.69160268ea309aa55ece0cd8528b941d., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,923 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1730976232769.69160268ea309aa55ece0cd8528b941d., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,925 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1730976232769.69160268ea309aa55ece0cd8528b941d., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,926 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uux', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1730976232769.69160268ea309aa55ece0cd8528b941d., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,928 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1730976232769.69160268ea309aa55ece0cd8528b941d., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:55,969 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,971 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,972 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,974 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:55,975 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:56,013 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='www', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1730976232769.1dd5c615432f6d3976b7d3e591d0a96a., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:56,015 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wwx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1730976232769.1dd5c615432f6d3976b7d3e591d0a96a., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:56,016 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wwy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1730976232769.1dd5c615432f6d3976b7d3e591d0a96a., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:56,017 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wwz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1730976232769.1dd5c615432f6d3976b7d3e591d0a96a., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:56,019 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wxa', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1730976232769.1dd5c615432f6d3976b7d3e591d0a96a., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:56,057 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1730976232769.0b29f5d988381aa77506fa3b92944854., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:56,059 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1730976232769.0b29f5d988381aa77506fa3b92944854., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:56,061 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1730976232769.0b29f5d988381aa77506fa3b92944854., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:56,063 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xya', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1730976232769.0b29f5d988381aa77506fa3b92944854., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:56,065 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xyc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1730976232769.0b29f5d988381aa77506fa3b92944854., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:56,066 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xyb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1730976232769.0b29f5d988381aa77506fa3b92944854., hostname=10641b2c0a69,39013,1730976230620, seqNum=2] 2024-11-07T10:43:56,101 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yyy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:56,103 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yyz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:56,104 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yza', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:56,106 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yzb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:56,108 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yzc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:56,109 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yzd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644., hostname=10641b2c0a69,41401,1730976230578, seqNum=2] 2024-11-07T10:43:56,144 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='zzz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1730976232769.c71057666e459779c2fa5a4311f4e99c., hostname=10641b2c0a69,40573,1730976230476, seqNum=2] 2024-11-07T10:43:56,205 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T10:43:56,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41401 {}] regionserver.RSRpcServices(2610): Large batch operation detected (greater than 5000) (HBASE-18023). Requested Number of Rows: 6327 Client: jenkins//172.17.0.2 first region in multi=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba. 2024-11-07T10:43:56,218 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59520, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T10:43:56,219 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T10:43:56,230 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41162, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T10:43:56,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40573 {}] regionserver.RSRpcServices(2610): Large batch operation detected (greater than 5000) (HBASE-18023). Requested Number of Rows: 5625 Client: jenkins//172.17.0.2 first region in multi=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1730976232769.bcc3491e6200809e4c9c7d529da2c4cf. 2024-11-07T10:43:56,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39013 {}] regionserver.RSRpcServices(2610): Large batch operation detected (greater than 5000) (HBASE-18023). Requested Number of Rows: 5624 Client: jenkins//172.17.0.2 first region in multi=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1730976232769.dbe69e720cc4f2a4a4bc41171b6b3c32. 2024-11-07T10:43:57,125 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,, stopping at row=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true ,, for max=2147483647 with caching=100 2024-11-07T10:43:57,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41401 {}] regionserver.StoreScanner(1138): Switch to stream read (scanned=32802 bytes) of info 2024-11-07T10:43:57,385 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: true 2024-11-07T10:43:57,385 INFO [Time-limited test {}] assignment.AssignmentTestingUtil(82): KILL REGION SERVER 10641b2c0a69,41401,1730976230578 2024-11-07T10:43:57,385 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 10641b2c0a69,41401,1730976230578 2024-11-07T10:43:57,386 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 10641b2c0a69,41401,1730976230578: Simulated kill ***** 2024-11-07T10:43:57,386 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-07T10:43:57,389 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-07T10:43:57,391 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-07T10:43:57,391 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-07T10:43:57,392 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-07T10:43:57,392 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 651253360 }, "NonHeapMemoryUsage": { "committed": 157810688, "init": 7667712, "max": -1, "used": 154981336 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "10641b2c0a69" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "10641b2c0a69" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "10641b2c0a69" } ] } 2024-11-07T10:43:57,394 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43605 {}] master.MasterRpcServices(700): 10641b2c0a69,41401,1730976230578 reported a fatal error: ***** ABORTING region server 10641b2c0a69,41401,1730976230578: Simulated kill ***** 2024-11-07T10:43:57,396 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '10641b2c0a69,41401,1730976230578' ***** 2024-11-07T10:43:57,396 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-07T10:43:57,397 INFO [RS:1;10641b2c0a69:41401 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-07T10:43:57,397 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T10:43:57,397 INFO [RS:1;10641b2c0a69:41401 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-07T10:43:57,397 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-07T10:43:57,397 INFO [RS:1;10641b2c0a69:41401 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-07T10:43:57,397 INFO [RS:1;10641b2c0a69:41401 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T10:43:57,398 INFO [RS:1;10641b2c0a69:41401 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;10641b2c0a69:41401. 2024-11-07T10:43:57,398 DEBUG [RS:1;10641b2c0a69:41401 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T10:43:57,404 DEBUG [RS:1;10641b2c0a69:41401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T10:43:57,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741836_1012 (size=54214) 2024-11-07T10:43:57,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741836_1012 (size=54214) 2024-11-07T10:43:57,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741836_1012 (size=54214) 2024-11-07T10:43:57,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741835_1011 (size=334915) 2024-11-07T10:43:57,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741835_1011 (size=334915) 2024-11-07T10:43:57,414 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578/10641b2c0a69%2C41401%2C1730976230578.1730976231927 not finished, retry = 0 2024-11-07T10:43:57,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741835_1011 (size=334915) 2024-11-07T10:43:57,493 INFO [regionserver/10641b2c0a69:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T10:43:57,515 DEBUG [RS:1;10641b2c0a69:41401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T10:43:57,515 INFO [RS:1;10641b2c0a69:41401 {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T10:43:57,516 INFO [RS:1;10641b2c0a69:41401 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41401 2024-11-07T10:43:57,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41401-0x10019d731180002, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/10641b2c0a69,41401,1730976230578 2024-11-07T10:43:57,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T10:43:57,524 INFO [RS:1;10641b2c0a69:41401 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T10:43:57,525 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [10641b2c0a69,41401,1730976230578] 2024-11-07T10:43:57,527 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/10641b2c0a69,41401,1730976230578 already deleted, retry=false 2024-11-07T10:43:57,527 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 10641b2c0a69,41401,1730976230578 on 10641b2c0a69,43605,1730976229810 2024-11-07T10:43:57,533 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true 2024-11-07T10:43:57,536 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=57 for 10641b2c0a69,41401,1730976230578 (carryingMeta=true) 10641b2c0a69,41401,1730976230578/CRASHED/regionCount=10/lock=java.util.concurrent.locks.ReentrantReadWriteLock@2f97e148[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-07T10:43:57,537 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true 2024-11-07T10:43:57,537 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-07T10:43:57,537 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true 2024-11-07T10:43:57,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T10:43:57,538 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T10:43:57,598 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-07T10:43:57,600 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 0 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true 2024-11-07T10:43:57,601 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T10:43:57,601 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T10:43:57,601 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T10:43:57,601 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T10:43:57,601 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T10:43:57,601 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T10:43:57,602 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T10:43:57,602 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T10:43:57,603 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T10:43:57,607 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T10:43:57,607 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43605 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T10:43:57,608 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T10:43:57,610 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-07T10:43:57,610 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true 2024-11-07T10:43:57,610 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T10:43:57,611 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-07T10:43:57,613 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:43:57,613 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=10641b2c0a69,41401,1730976230578, table=hbase:meta, region=1588230740 2024-11-07T10:43:57,613 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 10641b2c0a69,41401,1730976230578, state=OPEN 2024-11-07T10:43:57,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:43:57,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:43:57,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:43:57,615 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:43:57,615 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:43:57,615 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:43:57,617 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730976267617 2024-11-07T10:43:57,617 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-07T10:43:57,617 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true 2024-11-07T10:43:57,617 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-07T10:43:57,619 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 10641b2c0a69:41401 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T10:43:57,621 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:43:57,622 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-07T10:43:57,622 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:43:57,623 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 10641b2c0a69:41401 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:43:57,624 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true, isMeta: true 2024-11-07T10:43:57,626 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578-splitting 2024-11-07T10:43:57,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41401-0x10019d731180002, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T10:43:57,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41401-0x10019d731180002, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T10:43:57,627 INFO [RS:1;10641b2c0a69:41401 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T10:43:57,627 INFO [RS:1;10641b2c0a69:41401 {}] regionserver.HRegionServer(1031): Exiting; stopping=10641b2c0a69,41401,1730976230578; zookeeper connection closed. 2024-11-07T10:43:57,628 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@683d0d56 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@683d0d56 2024-11-07T10:43:57,628 INFO [PEWorker-1 {}] master.SplitWALManager(105): 10641b2c0a69,41401,1730976230578 WAL count=1, meta=true 2024-11-07T10:43:57,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta}] 2024-11-07T10:43:57,630 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-07T10:43:57,630 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true 2024-11-07T10:43:57,630 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T10:43:57,630 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T10:43:57,726 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:43:57,727 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:43:57,727 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 10641b2c0a69:41401 this server is in the failed servers list 2024-11-07T10:43:57,728 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:43:57,728 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:43:57,728 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:43:57,763 INFO [regionserver/10641b2c0a69:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-07T10:43:57,763 INFO [regionserver/10641b2c0a69:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-07T10:43:57,905 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T10:43:57,936 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:43:57,937 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:43:57,937 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 10641b2c0a69:41401 this server is in the failed servers list 2024-11-07T10:43:57,937 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:43:57,937 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:43:57,937 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:43:57,967 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true' 2024-11-07T10:43:57,970 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-07T10:43:58,246 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:43:58,247 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:43:58,247 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 10641b2c0a69:41401 this server is in the failed servers list 2024-11-07T10:43:58,247 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:43:58,247 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:43:58,247 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:43:58,757 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:43:58,757 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:43:58,757 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 10641b2c0a69:41401 this server is in the failed servers list 2024-11-07T10:43:58,757 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:43:58,757 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:43:58,757 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:43:59,777 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:43:59,777 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:43:59,778 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 10641b2c0a69:41401 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T10:43:59,781 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 10641b2c0a69,41401,1730976230578 for '' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 60000 ms, time elapsed = 2161 ms java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-07T10:43:59,782 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:43:59,782 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-07T10:43:59,782 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:43:59,782 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 10641b2c0a69:41401 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:44:00,212 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-07T10:44:00,212 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-07T10:44:00,214 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-07T10:44:00,214 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true Metrics about Tables on a single HBase RegionServer 2024-11-07T10:44:00,214 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T10:44:00,214 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-07T10:44:00,215 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-07T10:44:00,215 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-07T10:44:01,807 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:44:01,807 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:44:01,808 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 10641b2c0a69:41401 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T10:44:01,808 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 10641b2c0a69,41401,1730976230578 for '' in 1588230740 of hbase:meta failed, tries = 7, maxAttempts = 46, timeout = 60000 ms, time elapsed = 4190 ms java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-07T10:44:01,809 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:44:01,809 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-07T10:44:01,809 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:44:01,809 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 10641b2c0a69:41401 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:44:05,847 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:44:05,847 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:44:05,848 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 10641b2c0a69:41401 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T10:44:05,848 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 10641b2c0a69,41401,1730976230578 for '' in 1588230740 of hbase:meta failed, tries = 8, maxAttempts = 46, timeout = 60000 ms, time elapsed = 8230 ms java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-07T10:44:05,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:44:05,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-07T10:44:05,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:44:05,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 10641b2c0a69:41401 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:44:15,877 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:44:15,877 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:44:15,878 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 10641b2c0a69:41401 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T10:44:15,879 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 10641b2c0a69,41401,1730976230578 for '' in 1588230740 of hbase:meta failed, tries = 9, maxAttempts = 46, timeout = 60000 ms, time elapsed = 18261 ms java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-07T10:44:15,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:44:15,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-07T10:44:15,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:44:15,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 10641b2c0a69:41401 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:44:18,926 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T10:44:24,458 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=10641b2c0a69:43605 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-07T10:44:25,937 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:44:25,937 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:44:25,938 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 10641b2c0a69:41401 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T10:44:25,938 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 10641b2c0a69,41401,1730976230578 for '' in 1588230740 of hbase:meta failed, tries = 10, maxAttempts = 46, timeout = 60000 ms, time elapsed = 28320 ms java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-07T10:44:25,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:44:25,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-07T10:44:25,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:44:25,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 10641b2c0a69:41401 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:44:32,773 INFO [master/10641b2c0a69:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-07T10:44:32,773 INFO [master/10641b2c0a69:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-07T10:44:35,997 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:44:35,997 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:44:35,998 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 10641b2c0a69:41401 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T10:44:35,999 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 10641b2c0a69,41401,1730976230578 for '' in 1588230740 of hbase:meta failed, tries = 11, maxAttempts = 46, timeout = 60000 ms, time elapsed = 38381 ms java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-07T10:44:35,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:44:35,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-07T10:44:35,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:44:35,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 10641b2c0a69:41401 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:44:36,291 WARN [RS:0;10641b2c0a69:40573 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of 10641b2c0a69,41401,1730976230578: status=status unset, state=RUNNING, startTime=1730976237537, completionTime=-1 appears to have been leaked 2024-11-07T10:44:46,077 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:44:46,077 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:44:46,078 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 10641b2c0a69:41401 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T10:44:46,078 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 10641b2c0a69,41401,1730976230578 for '' in 1588230740 of hbase:meta failed, tries = 12, maxAttempts = 46, timeout = 60000 ms, time elapsed = 48460 ms java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-07T10:44:46,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:44:46,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-07T10:44:46,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:44:46,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 10641b2c0a69:41401 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:44:48,927 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T10:44:53,068 INFO [regionserver/10641b2c0a69:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: ExecutorStatusChore was stopped 2024-11-07T10:44:53,156 DEBUG [master/10641b2c0a69:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region dbe69e720cc4f2a4a4bc41171b6b3c32 changed from -1.0 to 0.0, refreshing cache 2024-11-07T10:44:53,156 DEBUG [master/10641b2c0a69:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3f6021bbda5b8b7b00318a57f02cd77d changed from -1.0 to 0.0, refreshing cache 2024-11-07T10:44:53,156 DEBUG [master/10641b2c0a69:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f89575b73864cac8b7b164f417ef378f changed from -1.0 to 0.0, refreshing cache 2024-11-07T10:44:53,157 DEBUG [master/10641b2c0a69:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1baabb362104b46ecb78fe236391ca6f changed from -1.0 to 0.0, refreshing cache 2024-11-07T10:44:53,157 DEBUG [master/10641b2c0a69:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region dec39ef6d94f3a519151705fce17adc4 changed from -1.0 to 0.0, refreshing cache 2024-11-07T10:44:53,157 DEBUG [master/10641b2c0a69:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 64da674e3a9ec8e7458eb07874d9f204 changed from -1.0 to 0.0, refreshing cache 2024-11-07T10:44:53,157 DEBUG [master/10641b2c0a69:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 69160268ea309aa55ece0cd8528b941d changed from -1.0 to 0.0, refreshing cache 2024-11-07T10:44:53,157 DEBUG [master/10641b2c0a69:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0b29f5d988381aa77506fa3b92944854 changed from -1.0 to 0.0, refreshing cache 2024-11-07T10:44:53,157 DEBUG [master/10641b2c0a69:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region bcc3491e6200809e4c9c7d529da2c4cf changed from -1.0 to 0.0, refreshing cache 2024-11-07T10:44:53,157 DEBUG [master/10641b2c0a69:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ddeeab6f2e44d85eb1ee916d3c5e8e9f changed from -1.0 to 0.0, refreshing cache 2024-11-07T10:44:53,157 DEBUG [master/10641b2c0a69:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c3ab41a5354d389521b694050824788e changed from -1.0 to 0.0, refreshing cache 2024-11-07T10:44:53,157 DEBUG [master/10641b2c0a69:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0a7d18696a90cfed5af1fbdb2a706b65 changed from -1.0 to 0.0, refreshing cache 2024-11-07T10:44:53,157 DEBUG [master/10641b2c0a69:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9b830813c6b6c3ebcdb59061af10faad changed from -1.0 to 0.0, refreshing cache 2024-11-07T10:44:53,157 DEBUG [master/10641b2c0a69:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9fba8e9ebafee4e67bfa1295ac44d4a5 changed from -1.0 to 0.0, refreshing cache 2024-11-07T10:44:53,157 DEBUG [master/10641b2c0a69:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2cf36f48b84f67b77887428ec6ad5b31 changed from -1.0 to 0.0, refreshing cache 2024-11-07T10:44:53,158 DEBUG [master/10641b2c0a69:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1dd5c615432f6d3976b7d3e591d0a96a changed from -1.0 to 0.0, refreshing cache 2024-11-07T10:44:53,158 DEBUG [master/10641b2c0a69:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c71057666e459779c2fa5a4311f4e99c changed from -1.0 to 0.0, refreshing cache 2024-11-07T10:44:57,627 WARN [Time-limited test {}] procedure.MasterProcedureTestingUtility$3(130): Failed to load meta org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=13, exceptions: 2024-11-07T10:43:57.623Z, java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:43:57.728Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:43:57.937Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:43:58.247Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:43:58.758Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:43:59.782Z, java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:44:01.809Z, java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:44:05.849Z, java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:44:15.879Z, java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:44:25.938Z, java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:44:35.999Z, java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:44:46.079Z, java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:493) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:461) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:422) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.fullScanRegions(MetaTableAccessor.java:108) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionStateStore.visitMeta(RegionStateStore.java:108) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.loadMeta(AssignmentManager.java:1907) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.joinCluster(AssignmentManager.java:1769) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:121) ~[test-classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:143) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:477) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:486) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPBase.testRecoveryAndDoubleExecution(TestSCPBase.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta.testRecoveryAndDoubleExecutionOnRsWithMeta(TestSCPWithMeta.java:36) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:102) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T10:44:57,629 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 1 exec state=pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true 2024-11-07T10:44:57,629 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T10:44:57,629 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T10:44:57,630 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T10:44:57,630 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T10:44:57,630 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T10:44:57,630 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T10:44:57,630 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T10:44:57,630 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T10:44:57,631 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-07T10:44:57,633 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T10:44:57,633 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T10:44:57,634 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-07T10:44:57,635 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=false; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true 2024-11-07T10:44:57,635 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=false; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T10:44:57,635 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-07T10:44:57,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:44:57,636 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=10641b2c0a69,41401,1730976230578, table=hbase:meta, region=1588230740 2024-11-07T10:44:57,636 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 10641b2c0a69,41401,1730976230578, state=OPEN 2024-11-07T10:44:57,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:44:57,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:44:57,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:44:57,638 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:44:57,638 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:44:57,638 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:44:57,641 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730976327641 2024-11-07T10:44:57,641 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-07T10:44:57,641 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true, isMeta: true 2024-11-07T10:44:57,641 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:44:57,641 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:44:57,642 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 10641b2c0a69:41401 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T10:44:57,643 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:44:57,643 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-07T10:44:57,643 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:44:57,643 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 10641b2c0a69:41401 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:44:57,644 INFO [PEWorker-1 {}] master.SplitWALManager(105): 10641b2c0a69,41401,1730976230578 WAL count=1, meta=true 2024-11-07T10:44:57,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta}] 2024-11-07T10:44:57,644 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-07T10:44:57,650 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=10641b2c0a69,40573,1730976230476 2024-11-07T10:44:57,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476}] 2024-11-07T10:44:57,747 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:44:57,747 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:44:57,747 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 10641b2c0a69:41401 this server is in the failed servers list 2024-11-07T10:44:57,747 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:44:57,748 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:44:57,748 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:44:57,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-11-07T10:44:57,838 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578-splitting/10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, size=52.9 K (54214bytes) 2024-11-07T10:44:57,839 DEBUG [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-07T10:44:57,839 DEBUG [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-07T10:44:57,839 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578-splitting/10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta 2024-11-07T10:44:57,844 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578-splitting/10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta after 4ms 2024-11-07T10:44:57,852 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578-splitting/10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta took 15ms 2024-11-07T10:44:57,862 DEBUG [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 2024-11-07T10:44:57,868 DEBUG [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578-splitting/10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta so closing down 2024-11-07T10:44:57,868 DEBUG [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-07T10:44:57,869 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta.temp 2024-11-07T10:44:57,873 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta.temp 2024-11-07T10:44:57,876 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-11-07T10:44:57,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741864_1040 (size=53973) 2024-11-07T10:44:57,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741864_1040 (size=53973) 2024-11-07T10:44:57,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741864_1040 (size=53973) 2024-11-07T10:44:57,889 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) 2024-11-07T10:44:57,891 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta.temp to hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-07T10:44:57,896 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 38 ms; skipped=1; WAL=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578-splitting/10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, size=52.9 K, length=54214, corrupted=false, cancelled=false 2024-11-07T10:44:57,896 DEBUG [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578-splitting/10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, journal: Splitting hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578-splitting/10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, size=52.9 K (54214bytes) at 1730976297838Finishing writing output for hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578-splitting/10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta so closing down at 1730976297868 (+30 ms)Creating recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta.temp at 1730976297873 (+5 ms)3 split writer threads finished at 1730976297876 (+3 ms)Closed recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) at 1730976297889 (+13 ms)Rename recovered edits hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta.temp to hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1730976297891 (+2 ms)Processed 58 edits across 1 Regions in 38 ms; skipped=1; WAL=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578-splitting/10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, size=52.9 K, length=54214, corrupted=false, cancelled=false at 1730976297896 (+5 ms) 2024-11-07T10:44:57,896 DEBUG [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578-splitting/10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta 2024-11-07T10:44:57,898 DEBUG [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-07T10:44:57,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43605 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-07T10:44:57,907 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578-splitting/10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta to hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/oldWALs 2024-11-07T10:44:57,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-07T10:44:57,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476 in 255 msec 2024-11-07T10:44:57,911 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=10641b2c0a69,40573,1730976230476 2024-11-07T10:44:57,914 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-07T10:44:57,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476 in 268 msec 2024-11-07T10:44:57,917 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578-splitting dir is empty, no logs to split. 2024-11-07T10:44:57,917 INFO [PEWorker-1 {}] master.SplitWALManager(105): 10641b2c0a69,41401,1730976230578 WAL count=0, meta=true 2024-11-07T10:44:57,917 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 10641b2c0a69,41401,1730976230578 WAL splitting is done? wals=0, meta=true 2024-11-07T10:44:57,917 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-07T10:44:57,917 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true 2024-11-07T10:44:57,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T10:44:57,917 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T10:44:57,957 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:44:57,957 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:44:57,958 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 10641b2c0a69:41401 this server is in the failed servers list 2024-11-07T10:44:57,958 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:44:57,958 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:44:57,958 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:44:58,266 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:44:58,267 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:44:58,267 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 10641b2c0a69:41401 this server is in the failed servers list 2024-11-07T10:44:58,267 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:44:58,268 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:44:58,268 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:44:58,460 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=10641b2c0a69:43605 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-07T10:44:58,777 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:44:58,777 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:44:58,777 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 10641b2c0a69:41401 this server is in the failed servers list 2024-11-07T10:44:58,778 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:44:58,778 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:44:58,778 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:44:59,796 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:44:59,797 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:44:59,797 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 10641b2c0a69:41401 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T10:44:59,798 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 10641b2c0a69,41401,1730976230578 for '' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 60000 ms, time elapsed = 2156 ms java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-07T10:44:59,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:44:59,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-07T10:44:59,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:44:59,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 10641b2c0a69:41401 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:45:01,817 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:45:01,817 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:45:01,818 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 10641b2c0a69:41401 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T10:45:01,818 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 10641b2c0a69,41401,1730976230578 for '' in 1588230740 of hbase:meta failed, tries = 7, maxAttempts = 46, timeout = 60000 ms, time elapsed = 4176 ms java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-07T10:45:01,819 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:45:01,819 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-07T10:45:01,819 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:45:01,819 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 10641b2c0a69:41401 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:45:05,857 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:45:05,857 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:45:05,858 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 10641b2c0a69:41401 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T10:45:05,858 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 10641b2c0a69,41401,1730976230578 for '' in 1588230740 of hbase:meta failed, tries = 8, maxAttempts = 46, timeout = 60000 ms, time elapsed = 8216 ms java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-07T10:45:05,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:45:05,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-07T10:45:05,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:45:05,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 10641b2c0a69:41401 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:45:15,946 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:45:15,946 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:45:15,947 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 10641b2c0a69:41401 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T10:45:15,948 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 10641b2c0a69,41401,1730976230578 for '' in 1588230740 of hbase:meta failed, tries = 9, maxAttempts = 46, timeout = 60000 ms, time elapsed = 18306 ms java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-07T10:45:15,948 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:45:15,948 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-07T10:45:15,948 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:45:15,948 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 10641b2c0a69:41401 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:45:18,927 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T10:45:26,057 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:45:26,057 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:45:26,058 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 10641b2c0a69:41401 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T10:45:26,058 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 10641b2c0a69,41401,1730976230578 for '' in 1588230740 of hbase:meta failed, tries = 10, maxAttempts = 46, timeout = 60000 ms, time elapsed = 28417 ms java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-07T10:45:26,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:45:26,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-07T10:45:26,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:45:26,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 10641b2c0a69:41401 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:45:36,107 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:45:36,107 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:45:36,108 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 10641b2c0a69:41401 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T10:45:36,109 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 10641b2c0a69,41401,1730976230578 for '' in 1588230740 of hbase:meta failed, tries = 11, maxAttempts = 46, timeout = 60000 ms, time elapsed = 38467 ms java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-07T10:45:36,109 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:45:36,109 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-07T10:45:36,109 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:45:36,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 10641b2c0a69:41401 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:45:46,206 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:45:46,206 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:45:46,207 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 10641b2c0a69:41401 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T10:45:46,208 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 10641b2c0a69,41401,1730976230578 for '' in 1588230740 of hbase:meta failed, tries = 12, maxAttempts = 46, timeout = 60000 ms, time elapsed = 48566 ms java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-07T10:45:46,208 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:45:46,208 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-07T10:45:46,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:45:46,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 10641b2c0a69:41401 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:45:48,927 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T10:45:51,757 INFO [regionserver/10641b2c0a69:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactedHFilesCleaner was stopped 2024-11-07T10:45:51,772 WARN [regionserver/10641b2c0a69:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 26, running: 0 2024-11-07T10:45:57,648 WARN [Time-limited test {}] procedure.MasterProcedureTestingUtility$3(130): Failed to load meta org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=13, exceptions: 2024-11-07T10:44:57.643Z, java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:44:57.748Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:44:57.958Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:44:58.268Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:44:58.778Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:44:59.798Z, java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:45:01.819Z, java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:45:05.859Z, java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:45:15.948Z, java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:45:26.059Z, java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:45:36.109Z, java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:45:46.209Z, java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:493) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:461) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:422) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.fullScanRegions(MetaTableAccessor.java:108) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionStateStore.visitMeta(RegionStateStore.java:108) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.loadMeta(AssignmentManager.java:1907) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.joinCluster(AssignmentManager.java:1769) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:121) ~[test-classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:143) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:477) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:486) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPBase.testRecoveryAndDoubleExecution(TestSCPBase.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta.testRecoveryAndDoubleExecutionOnRsWithMeta(TestSCPWithMeta.java:36) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:102) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T10:45:57,650 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 2 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true 2024-11-07T10:45:57,650 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T10:45:57,650 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T10:45:57,650 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T10:45:57,650 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T10:45:57,651 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T10:45:57,651 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T10:45:57,651 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T10:45:57,652 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T10:45:57,654 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-07T10:45:57,660 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T10:45:57,660 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T10:45:57,660 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T10:45:57,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T10:45:57,662 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-07T10:45:57,662 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=false; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true 2024-11-07T10:45:57,662 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476 2024-11-07T10:45:57,662 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476 2024-11-07T10:45:57,662 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=false; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T10:45:57,663 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:45:57,663 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:45:57,663 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-07T10:45:57,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:45:57,664 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=10641b2c0a69,41401,1730976230578, table=hbase:meta, region=1588230740 2024-11-07T10:45:57,664 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 10641b2c0a69,41401,1730976230578, state=OPEN 2024-11-07T10:45:57,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:45:57,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:45:57,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:45:57,669 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:45:57,669 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:45:57,669 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:45:57,685 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730976387685 2024-11-07T10:45:57,685 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-07T10:45:57,685 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:45:57,685 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:45:57,687 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 10641b2c0a69:41401 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T10:45:57,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:45:57,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-07T10:45:57,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:45:57,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 10641b2c0a69:41401 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:45:57,689 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578-splitting dir is empty, no logs to split. 2024-11-07T10:45:57,689 INFO [PEWorker-1 {}] master.SplitWALManager(105): 10641b2c0a69,41401,1730976230578 WAL count=0, meta=true 2024-11-07T10:45:57,689 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 10641b2c0a69,41401,1730976230578 WAL splitting is done? wals=0, meta=true 2024-11-07T10:45:57,689 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-07T10:45:57,692 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-07T10:45:57,692 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-07T10:45:57,692 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_GET_REGIONS, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true 2024-11-07T10:45:57,692 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T10:45:57,692 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_GET_REGIONS, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T10:45:57,797 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:45:57,797 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:45:57,798 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 10641b2c0a69:41401 this server is in the failed servers list 2024-11-07T10:45:57,798 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:45:57,798 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:45:57,798 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:45:58,007 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:45:58,007 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:45:58,008 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 10641b2c0a69:41401 this server is in the failed servers list 2024-11-07T10:45:58,008 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:45:58,008 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:45:58,008 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:45:58,317 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:45:58,317 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:45:58,317 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 10641b2c0a69:41401 this server is in the failed servers list 2024-11-07T10:45:58,318 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:45:58,318 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:45:58,318 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:45:58,827 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:45:58,827 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:45:58,828 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 10641b2c0a69:41401 this server is in the failed servers list 2024-11-07T10:45:58,828 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:45:58,828 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:45:58,828 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:45:59,837 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:45:59,837 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:45:59,838 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 10641b2c0a69:41401 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T10:45:59,838 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 10641b2c0a69,41401,1730976230578 for '' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 60000 ms, time elapsed = 2152 ms java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-07T10:45:59,839 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:45:59,839 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-07T10:45:59,839 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:45:59,839 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 10641b2c0a69:41401 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:46:01,857 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:46:01,857 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:46:01,858 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 10641b2c0a69:41401 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T10:46:01,858 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 10641b2c0a69,41401,1730976230578 for '' in 1588230740 of hbase:meta failed, tries = 7, maxAttempts = 46, timeout = 60000 ms, time elapsed = 4172 ms java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-07T10:46:01,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:46:01,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-07T10:46:01,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:46:01,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 10641b2c0a69:41401 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:46:05,461 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=10641b2c0a69:43605 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-07T10:46:05,887 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:46:05,887 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:46:05,888 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 10641b2c0a69:41401 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T10:46:05,888 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 10641b2c0a69,41401,1730976230578 for '' in 1588230740 of hbase:meta failed, tries = 8, maxAttempts = 46, timeout = 60000 ms, time elapsed = 8202 ms java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-07T10:46:05,888 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:46:05,888 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-07T10:46:05,888 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:46:05,889 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 10641b2c0a69:41401 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:46:15,916 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:46:15,916 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:46:15,917 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 10641b2c0a69:41401 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T10:46:15,918 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 10641b2c0a69,41401,1730976230578 for '' in 1588230740 of hbase:meta failed, tries = 9, maxAttempts = 46, timeout = 60000 ms, time elapsed = 18232 ms java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-07T10:46:15,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:46:15,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-07T10:46:15,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:46:15,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 10641b2c0a69:41401 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:46:18,927 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T10:46:25,997 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:46:25,997 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:46:25,998 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 10641b2c0a69:41401 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T10:46:25,999 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 10641b2c0a69,41401,1730976230578 for '' in 1588230740 of hbase:meta failed, tries = 10, maxAttempts = 46, timeout = 60000 ms, time elapsed = 28313 ms java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-07T10:46:25,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:46:25,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-07T10:46:25,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:46:25,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 10641b2c0a69:41401 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:46:36,027 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:46:36,027 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:46:36,028 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 10641b2c0a69:41401 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T10:46:36,029 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 10641b2c0a69,41401,1730976230578 for '' in 1588230740 of hbase:meta failed, tries = 11, maxAttempts = 46, timeout = 60000 ms, time elapsed = 38343 ms java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-07T10:46:36,029 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:46:36,029 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-07T10:46:36,029 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:46:36,029 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 10641b2c0a69:41401 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:46:39,599 WARN [RS:2;10641b2c0a69:39013 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of 10641b2c0a69,41401,1730976230578: status=Processing ServerCrashProcedure of 10641b2c0a69,41401,1730976230578 current State SERVER_CRASH_SPLIT_META_LOGS, state=RUNNING, startTime=1730976237610, completionTime=-1 appears to have been leaked 2024-11-07T10:46:39,599 WARN [RS:2;10641b2c0a69:39013 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of 10641b2c0a69,41401,1730976230578: status=Processing ServerCrashProcedure of 10641b2c0a69,41401,1730976230578 current State SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, state=RUNNING, startTime=1730976297634, completionTime=-1 appears to have been leaked 2024-11-07T10:46:46,047 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:46:46,047 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:46:46,048 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 10641b2c0a69:41401 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T10:46:46,049 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 10641b2c0a69,41401,1730976230578 for '' in 1588230740 of hbase:meta failed, tries = 12, maxAttempts = 46, timeout = 60000 ms, time elapsed = 48363 ms java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-07T10:46:46,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:46:46,050 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-07T10:46:46,050 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:46:46,050 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 10641b2c0a69:41401 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:46:48,928 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-07T10:46:57,687 WARN [Time-limited test {}] procedure.MasterProcedureTestingUtility$3(130): Failed to load meta org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=13, exceptions: 2024-11-07T10:45:57.687Z, java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:45:57.798Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:45:58.008Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:45:58.318Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:45:58.828Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:45:59.839Z, java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:46:01.859Z, java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:46:05.888Z, java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:46:15.918Z, java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:46:25.999Z, java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:46:36.029Z, java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:46:46.050Z, java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:493) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:461) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:422) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.fullScanRegions(MetaTableAccessor.java:108) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionStateStore.visitMeta(RegionStateStore.java:108) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.loadMeta(AssignmentManager.java:1907) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.joinCluster(AssignmentManager.java:1769) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:121) ~[test-classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:143) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:477) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:486) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPBase.testRecoveryAndDoubleExecution(TestSCPBase.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta.testRecoveryAndDoubleExecutionOnRsWithMeta(TestSCPWithMeta.java:36) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:102) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-07T10:46:57,689 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 3 exec state=pid=57, state=WAITING:SERVER_CRASH_GET_REGIONS, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true 2024-11-07T10:46:57,689 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T10:46:57,689 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T10:46:57,689 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T10:46:57,689 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T10:46:57,689 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T10:46:57,689 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T10:46:57,689 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T10:46:57,689 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T10:46:57,690 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T10:46:57,692 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T10:46:57,693 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T10:46:57,693 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T10:46:57,693 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T10:46:57,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-07T10:46:57,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=false; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true 2024-11-07T10:46:57,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476 2024-11-07T10:46:57,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476 2024-11-07T10:46:57,694 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=false; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T10:46:57,695 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:46:57,695 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:46:57,695 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-07T10:46:57,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:46:57,696 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=10641b2c0a69,41401,1730976230578, table=hbase:meta, region=1588230740 2024-11-07T10:46:57,696 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 10641b2c0a69,41401,1730976230578, state=OPEN 2024-11-07T10:46:57,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:46:57,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:46:57,698 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:46:57,698 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:46:57,698 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:46:57,699 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:46:57,699 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730976447699 2024-11-07T10:46:57,700 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-07T10:46:57,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-07T10:46:57,700 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-07T10:46:57,700 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:46:57,700 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:46:57,701 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 10641b2c0a69:41401 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T10:46:57,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:46:57,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-07T10:46:57,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:46:57,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 10641b2c0a69:41401 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:46:57,702 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T10:46:57,703 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-07T10:46:57,807 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:46:57,807 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1] 2024-11-07T10:46:57,807 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 10641b2c0a69:41401 this server is in the failed servers list 2024-11-07T10:46:57,808 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:46:57,808 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:46:57,808 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:46:57,853 DEBUG [10641b2c0a69:43605 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-07T10:46:57,854 DEBUG [10641b2c0a69:43605 {}] balancer.BalancerClusterState(204): Hosts are {10641b2c0a69=0} racks are {/default-rack=0} 2024-11-07T10:46:57,854 DEBUG [10641b2c0a69:43605 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-07T10:46:57,854 DEBUG [10641b2c0a69:43605 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-07T10:46:57,854 DEBUG [10641b2c0a69:43605 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-07T10:46:57,854 DEBUG [10641b2c0a69:43605 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-07T10:46:57,854 INFO [10641b2c0a69:43605 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-07T10:46:57,854 INFO [10641b2c0a69:43605 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-07T10:46:57,854 DEBUG [10641b2c0a69:43605 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-07T10:46:57,854 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=10641b2c0a69,40573,1730976230476 2024-11-07T10:46:57,856 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 10641b2c0a69,40573,1730976230476, state=OPENING 2024-11-07T10:46:57,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:46:57,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:46:57,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:46:57,857 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:46:57,858 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:46:57,858 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:46:57,858 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T10:46:57,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=10641b2c0a69,40573,1730976230476}] 2024-11-07T10:46:58,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46895, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-07T10:46:58,017 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:46:58,017 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,40573,1730976230476, seqNum=-1] 2024-11-07T10:46:58,018 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-07T10:46:58,019 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48141, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T10:46:58,020 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-07T10:46:58,020 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-07T10:46:58,020 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-07T10:46:58,022 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=10641b2c0a69%2C40573%2C1730976230476.meta, suffix=.meta, logDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,40573,1730976230476, archiveDir=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/oldWALs, maxLogs=32 2024-11-07T10:46:58,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40573 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Scan size: 121 connection: 172.17.0.2:48141 deadline: 1730976477701, exception=org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on 10641b2c0a69,40573,1730976230476 2024-11-07T10:46:58,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,40573,1730976230476, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,40573,1730976230476, seqNum=-1, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on 10641b2c0a69,40573,1730976230476 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T10:46:58,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,40573,1730976230476, seqNum=-1 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on 10641b2c0a69,40573,1730976230476 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T10:46:58,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,40573,1730976230476, seqNum=-1 from cache 2024-11-07T10:46:58,044 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,40573,1730976230476/10641b2c0a69%2C40573%2C1730976230476.meta.1730976418024.meta, exclude list is [], retry=0 2024-11-07T10:46:58,048 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40045,DS-5d0e97fa-7109-4597-9d71-8fb432e0463c,DISK] 2024-11-07T10:46:58,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33357,DS-ca0d026c-750a-4349-8e6f-ce26c9f0979a,DISK] 2024-11-07T10:46:58,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41673,DS-4666ca76-3f5e-48c3-8d05-ac47a18a697b,DISK] 2024-11-07T10:46:58,055 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,40573,1730976230476/10641b2c0a69%2C40573%2C1730976230476.meta.1730976418024.meta 2024-11-07T10:46:58,057 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41569:41569),(127.0.0.1/127.0.0.1:34291:34291),(127.0.0.1/127.0.0.1:33799:33799)] 2024-11-07T10:46:58,057 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-07T10:46:58,057 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-07T10:46:58,058 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-07T10:46:58,058 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-07T10:46:58,058 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-07T10:46:58,058 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:46:58,058 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-07T10:46:58,058 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-07T10:46:58,060 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-07T10:46:58,061 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-07T10:46:58,062 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:46:58,062 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:46:58,063 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-07T10:46:58,064 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-07T10:46:58,064 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:46:58,064 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:46:58,064 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-07T10:46:58,065 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-07T10:46:58,065 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:46:58,065 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:46:58,066 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-07T10:46:58,066 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-07T10:46:58,066 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:46:58,067 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-07T10:46:58,067 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-07T10:46:58,067 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740 2024-11-07T10:46:58,070 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740 2024-11-07T10:46:58,070 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-07T10:46:58,088 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-07T10:46:58,092 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=49.58 KB heapSize=69.98 KB 2024-11-07T10:46:58,185 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/.tmp/info/f34ccee0e24e4cfaae58c2e815c5ff6b is 268, key is testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1730976232769.bcc3491e6200809e4c9c7d529da2c4cf./info:regioninfo/1730976233890/Put/seqid=0 2024-11-07T10:46:58,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741866_1042 (size=58053) 2024-11-07T10:46:58,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741866_1042 (size=58053) 2024-11-07T10:46:58,199 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.31 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/.tmp/info/f34ccee0e24e4cfaae58c2e815c5ff6b 2024-11-07T10:46:58,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741866_1042 (size=58053) 2024-11-07T10:46:58,280 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/.tmp/ns/1119f304f231433ca98dcdd6a78f444c is 43, key is default/ns:d/1730976232527/Put/seqid=0 2024-11-07T10:46:58,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741867_1043 (size=5153) 2024-11-07T10:46:58,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741867_1043 (size=5153) 2024-11-07T10:46:58,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741867_1043 (size=5153) 2024-11-07T10:46:58,310 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/.tmp/ns/1119f304f231433ca98dcdd6a78f444c 2024-11-07T10:46:58,337 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:46:58,337 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,40573,1730976230476, seqNum=-1] 2024-11-07T10:46:58,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40573 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 121 connection: 172.17.0.2:48141 deadline: 1730976477700, exception=org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 10641b2c0a69,40573,1730976230476 2024-11-07T10:46:58,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,40573,1730976230476, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,40573,1730976230476, seqNum=-1, error=org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 10641b2c0a69,40573,1730976230476 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T10:46:58,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,40573,1730976230476, seqNum=-1 is org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 10641b2c0a69,40573,1730976230476 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-07T10:46:58,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,40573,1730976230476, seqNum=-1 from cache 2024-11-07T10:46:58,381 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/.tmp/table/bd196597b2814249812bc55549b5cbf9 is 105, key is testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/table:state/1730976233957/Put/seqid=0 2024-11-07T10:46:58,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741868_1044 (size=5620) 2024-11-07T10:46:58,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741868_1044 (size=5620) 2024-11-07T10:46:58,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741868_1044 (size=5620) 2024-11-07T10:46:58,398 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=202 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/.tmp/table/bd196597b2814249812bc55549b5cbf9 2024-11-07T10:46:58,421 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/.tmp/info/f34ccee0e24e4cfaae58c2e815c5ff6b as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/info/f34ccee0e24e4cfaae58c2e815c5ff6b 2024-11-07T10:46:58,439 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/info/f34ccee0e24e4cfaae58c2e815c5ff6b, entries=260, sequenceid=60, filesize=56.7 K 2024-11-07T10:46:58,442 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/.tmp/ns/1119f304f231433ca98dcdd6a78f444c as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/ns/1119f304f231433ca98dcdd6a78f444c 2024-11-07T10:46:58,459 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/ns/1119f304f231433ca98dcdd6a78f444c, entries=2, sequenceid=60, filesize=5.0 K 2024-11-07T10:46:58,461 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/.tmp/table/bd196597b2814249812bc55549b5cbf9 as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/table/bd196597b2814249812bc55549b5cbf9 2024-11-07T10:46:58,472 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/table/bd196597b2814249812bc55549b5cbf9, entries=2, sequenceid=60, filesize=5.5 K 2024-11-07T10:46:58,477 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~49.58 KB/50772, heapSize ~69.69 KB/71360, currentSize=0 B/0 for 1588230740 in 384ms, sequenceid=60, compaction requested=false; wal=null 2024-11-07T10:46:58,478 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-07T10:46:58,480 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-07T10:46:58,480 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-07T10:46:58,481 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-07T10:46:58,483 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-07T10:46:58,486 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-07T10:46:58,487 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62957426, jitterRate=-0.061861246824264526}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-07T10:46:58,487 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-07T10:46:58,488 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1730976418058Writing region info on filesystem at 1730976418058Initializing all the Stores at 1730976418060 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730976418060Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730976418060Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976418060Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1730976418060Obtaining lock to block concurrent updates at 1730976418092 (+32 ms)Preparing flush snapshotting stores in 1588230740 at 1730976418092Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=50772, getHeapSize=71600, getOffHeapSize=0, getCellsCount=264 at 1730976418096 (+4 ms)Flushing stores of hbase:meta,,1.1588230740 at 1730976418096Flushing 1588230740/info: creating writer at 1730976418100 (+4 ms)Flushing 1588230740/info: appending metadata at 1730976418177 (+77 ms)Flushing 1588230740/info: closing flushed file at 1730976418181 (+4 ms)Flushing 1588230740/ns: creating writer at 1730976418256 (+75 ms)Flushing 1588230740/ns: appending metadata at 1730976418279 (+23 ms)Flushing 1588230740/ns: closing flushed file at 1730976418279Flushing 1588230740/table: creating writer at 1730976418329 (+50 ms)Flushing 1588230740/table: appending metadata at 1730976418380 (+51 ms)Flushing 1588230740/table: closing flushed file at 1730976418380Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5dd32718: reopening flushed file at 1730976418417 (+37 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60483b9a: reopening flushed file at 1730976418440 (+23 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e70de39: reopening flushed file at 1730976418459 (+19 ms)Finished flush of dataSize ~49.58 KB/50772, heapSize ~69.69 KB/71360, currentSize=0 B/0 for 1588230740 in 384ms, sequenceid=60, compaction requested=false; wal=null at 1730976418477 (+18 ms)Cleaning up temporary data from old regions at 1730976418480 (+3 ms)Running coprocessor post-open hooks at 1730976418487 (+7 ms)Region opened successfully at 1730976418487 2024-11-07T10:46:58,489 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1730976418011 2024-11-07T10:46:58,492 DEBUG [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-07T10:46:58,492 INFO [RS_OPEN_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-07T10:46:58,493 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=10641b2c0a69,40573,1730976230476 2024-11-07T10:46:58,494 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 10641b2c0a69,40573,1730976230476, state=OPEN 2024-11-07T10:46:58,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:46:58,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:46:58,496 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:46:58,496 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:46:58,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:46:58,497 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:46:58,497 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=10641b2c0a69,40573,1730976230476 2024-11-07T10:46:58,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-07T10:46:58,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=10641b2c0a69,40573,1730976230476 in 639 msec 2024-11-07T10:46:58,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-11-07T10:46:58,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 801 msec 2024-11-07T10:46:58,847 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:46:58,847 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,40573,1730976230476, seqNum=-1] 2024-11-07T10:46:58,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a68cc5b6ea7d9e2214ab42d65b16eba, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:46:58,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcc3491e6200809e4c9c7d529da2c4cf, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:46:58,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbe69e720cc4f2a4a4bc41171b6b3c32, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:46:58,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d4b381fbb1d4cd388f45774fca53b2eb, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:46:58,902 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f6021bbda5b8b7b00318a57f02cd77d, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:46:58,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddeeab6f2e44d85eb1ee916d3c5e8e9f, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:46:58,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db89a463448eb182960960c83f4e0b16, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:46:58,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3ab41a5354d389521b694050824788e, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:46:58,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f89575b73864cac8b7b164f417ef378f, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:46:58,903 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72bf61b738a2f9b35913a192fc32815b, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:46:58,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a7d18696a90cfed5af1fbdb2a706b65, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:46:58,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1baabb362104b46ecb78fe236391ca6f, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:46:58,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ecb70a5eda60ab5744089e2ffe59d71, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:46:58,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b830813c6b6c3ebcdb59061af10faad, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:46:58,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dec39ef6d94f3a519151705fce17adc4, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:46:58,904 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=541874b6b96014b10121ede508f1f61f, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:46:58,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64da674e3a9ec8e7458eb07874d9f204, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:46:58,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fba8e9ebafee4e67bfa1295ac44d4a5, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:46:58,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1701fae2bf725fdd94c16d1c6b8bc75f, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:46:58,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cf36f48b84f67b77887428ec6ad5b31, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:46:58,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69160268ea309aa55ece0cd8528b941d, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:46:58,905 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=257c2c76c61c100547acb4a778fb1fef, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:46:58,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1dd5c615432f6d3976b7d3e591d0a96a, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:46:58,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b29f5d988381aa77506fa3b92944854, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:46:58,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f449585c6b7b82f4442604100a49644, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:46:58,906 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c71057666e459779c2fa5a4311f4e99c, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:46:58,906 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-07T10:46:58,906 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730976478906 2024-11-07T10:46:58,906 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730976538906 2024-11-07T10:46:58,906 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 1.2060 sec 2024-11-07T10:46:58,907 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 10641b2c0a69,41401,1730976230578 had 10 regions 2024-11-07T10:46:58,907 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-07T10:46:58,907 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true 2024-11-07T10:46:58,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T10:46:58,908 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T10:46:58,908 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 4 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true 2024-11-07T10:46:58,908 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T10:46:58,908 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T10:46:58,908 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T10:46:58,908 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T10:46:58,908 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T10:46:58,909 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T10:46:58,909 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T10:46:58,909 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T10:46:58,910 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-07T10:46:58,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T10:46:58,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T10:46:58,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T10:46:58,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T10:46:58,912 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T10:46:58,912 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T10:46:58,913 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-07T10:46:58,914 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_GET_REGIONS, hasLock=false; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true 2024-11-07T10:46:58,914 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476 2024-11-07T10:46:58,914 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476 2024-11-07T10:46:58,914 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T10:46:58,914 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=10641b2c0a69,40573,1730976230476 2024-11-07T10:46:58,915 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_GET_REGIONS, hasLock=false; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T10:46:58,915 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:46:58,915 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:46:58,915 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:46:58,915 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=10641b2c0a69,40573,1730976230476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:46:58,915 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-07T10:46:58,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=61 2024-11-07T10:46:58,917 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=10641b2c0a69,40573,1730976230476, table=hbase:meta, region=1588230740 2024-11-07T10:46:58,917 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 10641b2c0a69,40573,1730976230476, state=OPEN 2024-11-07T10:46:58,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:46:58,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:46:58,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:46:58,919 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:46:58,919 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:46:58,920 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:46:58,925 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730976448925 2024-11-07T10:46:58,925 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-07T10:46:58,934 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a68cc5b6ea7d9e2214ab42d65b16eba, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:46:58,934 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcc3491e6200809e4c9c7d529da2c4cf, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:46:58,934 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbe69e720cc4f2a4a4bc41171b6b3c32, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:46:58,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d4b381fbb1d4cd388f45774fca53b2eb, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:46:58,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f6021bbda5b8b7b00318a57f02cd77d, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:46:58,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddeeab6f2e44d85eb1ee916d3c5e8e9f, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:46:58,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db89a463448eb182960960c83f4e0b16, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:46:58,935 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3ab41a5354d389521b694050824788e, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:46:58,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f89575b73864cac8b7b164f417ef378f, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:46:58,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72bf61b738a2f9b35913a192fc32815b, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:46:58,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a7d18696a90cfed5af1fbdb2a706b65, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:46:58,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1baabb362104b46ecb78fe236391ca6f, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:46:58,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ecb70a5eda60ab5744089e2ffe59d71, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:46:58,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b830813c6b6c3ebcdb59061af10faad, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:46:58,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dec39ef6d94f3a519151705fce17adc4, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:46:58,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=541874b6b96014b10121ede508f1f61f, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:46:58,936 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64da674e3a9ec8e7458eb07874d9f204, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:46:58,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fba8e9ebafee4e67bfa1295ac44d4a5, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:46:58,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1701fae2bf725fdd94c16d1c6b8bc75f, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:46:58,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cf36f48b84f67b77887428ec6ad5b31, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:46:58,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69160268ea309aa55ece0cd8528b941d, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:46:58,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=257c2c76c61c100547acb4a778fb1fef, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:46:58,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1dd5c615432f6d3976b7d3e591d0a96a, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:46:58,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b29f5d988381aa77506fa3b92944854, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:46:58,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f449585c6b7b82f4442604100a49644, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:46:58,937 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c71057666e459779c2fa5a4311f4e99c, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:46:58,937 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-07T10:46:58,937 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730976478937 2024-11-07T10:46:58,937 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730976538937 2024-11-07T10:46:58,937 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-07T10:46:58,938 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 10641b2c0a69,41401,1730976230578 had 9 regions 2024-11-07T10:46:58,938 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-07T10:46:58,939 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true, isMeta: false 2024-11-07T10:46:58,941 INFO [PEWorker-1 {}] master.SplitWALManager(105): 10641b2c0a69,41401,1730976230578 WAL count=1, meta=false 2024-11-07T10:46:58,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 10641b2c0a69%2C41401%2C1730976230578.1730976231927}] 2024-11-07T10:46:58,942 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-07T10:46:58,942 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true 2024-11-07T10:46:58,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T10:46:58,942 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T10:46:59,188 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 5 exec state=pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true 2024-11-07T10:46:59,188 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T10:46:59,188 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T10:46:59,188 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T10:46:59,188 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T10:46:59,188 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T10:46:59,189 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T10:46:59,189 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T10:46:59,189 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T10:46:59,190 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T10:46:59,192 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T10:46:59,192 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T10:46:59,192 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T10:46:59,192 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T10:46:59,192 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T10:46:59,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T10:46:59,193 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-07T10:46:59,194 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=false; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true 2024-11-07T10:46:59,194 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476 2024-11-07T10:46:59,194 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476 2024-11-07T10:46:59,194 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T10:46:59,194 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=10641b2c0a69,40573,1730976230476 2024-11-07T10:46:59,194 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=false; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T10:46:59,194 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:46:59,194 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:46:59,194 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:46:59,194 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=10641b2c0a69,40573,1730976230476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:46:59,194 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-07T10:46:59,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=61 2024-11-07T10:46:59,195 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=10641b2c0a69,40573,1730976230476, table=hbase:meta, region=1588230740 2024-11-07T10:46:59,196 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 10641b2c0a69,40573,1730976230476, state=OPEN 2024-11-07T10:46:59,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:46:59,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:46:59,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:46:59,198 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:46:59,198 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:46:59,198 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:46:59,204 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730976449204 2024-11-07T10:46:59,204 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-07T10:46:59,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a68cc5b6ea7d9e2214ab42d65b16eba, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:46:59,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcc3491e6200809e4c9c7d529da2c4cf, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:46:59,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbe69e720cc4f2a4a4bc41171b6b3c32, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:46:59,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d4b381fbb1d4cd388f45774fca53b2eb, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:46:59,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f6021bbda5b8b7b00318a57f02cd77d, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:46:59,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddeeab6f2e44d85eb1ee916d3c5e8e9f, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:46:59,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db89a463448eb182960960c83f4e0b16, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:46:59,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3ab41a5354d389521b694050824788e, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:46:59,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f89575b73864cac8b7b164f417ef378f, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:46:59,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72bf61b738a2f9b35913a192fc32815b, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:46:59,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a7d18696a90cfed5af1fbdb2a706b65, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:46:59,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1baabb362104b46ecb78fe236391ca6f, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:46:59,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ecb70a5eda60ab5744089e2ffe59d71, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:46:59,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b830813c6b6c3ebcdb59061af10faad, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:46:59,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dec39ef6d94f3a519151705fce17adc4, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:46:59,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=541874b6b96014b10121ede508f1f61f, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:46:59,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64da674e3a9ec8e7458eb07874d9f204, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:46:59,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fba8e9ebafee4e67bfa1295ac44d4a5, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:46:59,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1701fae2bf725fdd94c16d1c6b8bc75f, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:46:59,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cf36f48b84f67b77887428ec6ad5b31, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:46:59,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69160268ea309aa55ece0cd8528b941d, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:46:59,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=257c2c76c61c100547acb4a778fb1fef, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:46:59,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1dd5c615432f6d3976b7d3e591d0a96a, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:46:59,213 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b29f5d988381aa77506fa3b92944854, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:46:59,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f449585c6b7b82f4442604100a49644, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:46:59,214 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c71057666e459779c2fa5a4311f4e99c, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:46:59,214 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-07T10:46:59,214 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730976479214 2024-11-07T10:46:59,214 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730976539214 2024-11-07T10:46:59,214 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-07T10:46:59,218 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true, isMeta: false 2024-11-07T10:46:59,221 INFO [PEWorker-1 {}] master.SplitWALManager(105): 10641b2c0a69,41401,1730976230578 WAL count=1, meta=false 2024-11-07T10:46:59,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 10641b2c0a69%2C41401%2C1730976230578.1730976231927}] 2024-11-07T10:46:59,222 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-07T10:46:59,224 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=10641b2c0a69,40573,1730976230476 2024-11-07T10:46:59,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 10641b2c0a69%2C41401%2C1730976230578.1730976231927, worker=10641b2c0a69,40573,1730976230476}] 2024-11-07T10:46:59,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-11-07T10:46:59,403 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578-splitting/10641b2c0a69%2C41401%2C1730976230578.1730976231927, size=327.1 K (334915bytes) 2024-11-07T10:46:59,403 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578-splitting/10641b2c0a69%2C41401%2C1730976230578.1730976231927 2024-11-07T10:46:59,404 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578-splitting/10641b2c0a69%2C41401%2C1730976230578.1730976231927 after 1ms 2024-11-07T10:46:59,413 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578-splitting/10641b2c0a69%2C41401%2C1730976230578.1730976231927 took 10ms 2024-11-07T10:46:59,419 DEBUG [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for d4b381fbb1d4cd388f45774fca53b2eb: last_flushed_sequence_id: 18446744073709551615 2024-11-07T10:46:59,421 DEBUG [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 7a68cc5b6ea7d9e2214ab42d65b16eba: last_flushed_sequence_id: 18446744073709551615 2024-11-07T10:46:59,422 DEBUG [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 8f449585c6b7b82f4442604100a49644: last_flushed_sequence_id: 18446744073709551615 2024-11-07T10:46:59,425 DEBUG [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for db89a463448eb182960960c83f4e0b16: last_flushed_sequence_id: 18446744073709551615 2024-11-07T10:46:59,427 DEBUG [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 257c2c76c61c100547acb4a778fb1fef: last_flushed_sequence_id: 18446744073709551615 2024-11-07T10:46:59,428 DEBUG [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 541874b6b96014b10121ede508f1f61f: last_flushed_sequence_id: 18446744073709551615 2024-11-07T10:46:59,429 DEBUG [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 72bf61b738a2f9b35913a192fc32815b: last_flushed_sequence_id: 18446744073709551615 2024-11-07T10:46:59,431 DEBUG [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 8ecb70a5eda60ab5744089e2ffe59d71: last_flushed_sequence_id: 18446744073709551615 2024-11-07T10:46:59,432 DEBUG [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 1701fae2bf725fdd94c16d1c6b8bc75f: last_flushed_sequence_id: 18446744073709551615 2024-11-07T10:46:59,440 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp 2024-11-07T10:46:59,442 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp 2024-11-07T10:46:59,445 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp 2024-11-07T10:46:59,449 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7a68cc5b6ea7d9e2214ab42d65b16eba/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp 2024-11-07T10:46:59,452 DEBUG [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578-splitting/10641b2c0a69%2C41401%2C1730976230578.1730976231927 so closing down 2024-11-07T10:46:59,452 DEBUG [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-07T10:46:59,452 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp 2024-11-07T10:46:59,453 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d4b381fbb1d4cd388f45774fca53b2eb/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp 2024-11-07T10:46:59,459 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp 2024-11-07T10:46:59,470 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1701fae2bf725fdd94c16d1c6b8bc75f/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp 2024-11-07T10:46:59,473 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp 2024-11-07T10:46:59,479 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72bf61b738a2f9b35913a192fc32815b/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp 2024-11-07T10:46:59,485 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp 2024-11-07T10:46:59,487 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/257c2c76c61c100547acb4a778fb1fef/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp 2024-11-07T10:46:59,487 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/541874b6b96014b10121ede508f1f61f/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp 2024-11-07T10:46:59,489 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ecb70a5eda60ab5744089e2ffe59d71/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp 2024-11-07T10:46:59,490 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp 2024-11-07T10:46:59,492 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp 2024-11-07T10:46:59,492 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8f449585c6b7b82f4442604100a49644/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp 2024-11-07T10:46:59,496 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/db89a463448eb182960960c83f4e0b16/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp 2024-11-07T10:46:59,497 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-11-07T10:46:59,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741869_1045 (size=36767) 2024-11-07T10:46:59,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741869_1045 (size=36767) 2024-11-07T10:46:59,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741871_1047 (size=36767) 2024-11-07T10:46:59,527 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d4b381fbb1d4cd388f45774fca53b2eb/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp (wrote 1 edits, skipped 0 edits in 1 ms) 2024-11-07T10:46:59,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741871_1047 (size=36767) 2024-11-07T10:46:59,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741870_1046 (size=36767) 2024-11-07T10:46:59,529 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d4b381fbb1d4cd388f45774fca53b2eb/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp to hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d4b381fbb1d4cd388f45774fca53b2eb/recovered.edits/0000000000000000004 2024-11-07T10:46:59,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741871_1047 (size=36767) 2024-11-07T10:46:59,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741870_1046 (size=36767) 2024-11-07T10:46:59,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741869_1045 (size=36767) 2024-11-07T10:46:59,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741870_1046 (size=36767) 2024-11-07T10:46:59,531 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/db89a463448eb182960960c83f4e0b16/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-07T10:46:59,532 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7a68cc5b6ea7d9e2214ab42d65b16eba/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp (wrote 1 edits, skipped 0 edits in 1 ms) 2024-11-07T10:46:59,534 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/db89a463448eb182960960c83f4e0b16/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp to hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/db89a463448eb182960960c83f4e0b16/recovered.edits/0000000000000000004 2024-11-07T10:46:59,534 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7a68cc5b6ea7d9e2214ab42d65b16eba/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp to hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7a68cc5b6ea7d9e2214ab42d65b16eba/recovered.edits/0000000000000000004 2024-11-07T10:46:59,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741872_1048 (size=36767) 2024-11-07T10:46:59,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741872_1048 (size=36767) 2024-11-07T10:46:59,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741872_1048 (size=36767) 2024-11-07T10:46:59,556 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1701fae2bf725fdd94c16d1c6b8bc75f/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-07T10:46:59,557 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1701fae2bf725fdd94c16d1c6b8bc75f/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp to hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1701fae2bf725fdd94c16d1c6b8bc75f/recovered.edits/0000000000000000004 2024-11-07T10:46:59,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741873_1049 (size=36767) 2024-11-07T10:46:59,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741873_1049 (size=36767) 2024-11-07T10:46:59,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741873_1049 (size=36767) 2024-11-07T10:46:59,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741874_1050 (size=36767) 2024-11-07T10:46:59,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741874_1050 (size=36767) 2024-11-07T10:46:59,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741874_1050 (size=36767) 2024-11-07T10:46:59,585 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72bf61b738a2f9b35913a192fc32815b/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp (wrote 1 edits, skipped 0 edits in 1 ms) 2024-11-07T10:46:59,587 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72bf61b738a2f9b35913a192fc32815b/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp to hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72bf61b738a2f9b35913a192fc32815b/recovered.edits/0000000000000000004 2024-11-07T10:46:59,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741875_1051 (size=36767) 2024-11-07T10:46:59,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741875_1051 (size=36767) 2024-11-07T10:46:59,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741875_1051 (size=36767) 2024-11-07T10:46:59,625 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ecb70a5eda60ab5744089e2ffe59d71/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-07T10:46:59,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741876_1052 (size=36767) 2024-11-07T10:46:59,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741876_1052 (size=36767) 2024-11-07T10:46:59,629 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ecb70a5eda60ab5744089e2ffe59d71/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp to hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ecb70a5eda60ab5744089e2ffe59d71/recovered.edits/0000000000000000004 2024-11-07T10:46:59,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741876_1052 (size=36767) 2024-11-07T10:46:59,631 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/541874b6b96014b10121ede508f1f61f/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-07T10:46:59,633 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/541874b6b96014b10121ede508f1f61f/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp to hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/541874b6b96014b10121ede508f1f61f/recovered.edits/0000000000000000004 2024-11-07T10:46:59,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741877_1053 (size=36767) 2024-11-07T10:46:59,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741877_1053 (size=36767) 2024-11-07T10:46:59,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741877_1053 (size=36767) 2024-11-07T10:46:59,644 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8f449585c6b7b82f4442604100a49644/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-07T10:46:59,646 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8f449585c6b7b82f4442604100a49644/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp to hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8f449585c6b7b82f4442604100a49644/recovered.edits/0000000000000000004 2024-11-07T10:46:59,974 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/257c2c76c61c100547acb4a778fb1fef/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp (wrote 1 edits, skipped 0 edits in 1 ms) 2024-11-07T10:46:59,976 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/257c2c76c61c100547acb4a778fb1fef/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp to hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/257c2c76c61c100547acb4a778fb1fef/recovered.edits/0000000000000000004 2024-11-07T10:46:59,976 INFO [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 18 edits across 9 Regions in 560 ms; skipped=9; WAL=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578-splitting/10641b2c0a69%2C41401%2C1730976230578.1730976231927, size=327.1 K, length=334915, corrupted=false, cancelled=false 2024-11-07T10:46:59,976 DEBUG [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578-splitting/10641b2c0a69%2C41401%2C1730976230578.1730976231927, journal: Splitting hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578-splitting/10641b2c0a69%2C41401%2C1730976230578.1730976231927, size=327.1 K (334915bytes) at 1730976419403Creating recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7a68cc5b6ea7d9e2214ab42d65b16eba/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp at 1730976419449 (+46 ms)Finishing writing output for hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578-splitting/10641b2c0a69%2C41401%2C1730976230578.1730976231927 so closing down at 1730976419452 (+3 ms)Creating recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d4b381fbb1d4cd388f45774fca53b2eb/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp at 1730976419453 (+1 ms)Creating recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1701fae2bf725fdd94c16d1c6b8bc75f/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp at 1730976419470 (+17 ms)Creating recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72bf61b738a2f9b35913a192fc32815b/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp at 1730976419479 (+9 ms)Creating recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/257c2c76c61c100547acb4a778fb1fef/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp at 1730976419487 (+8 ms)Creating recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/541874b6b96014b10121ede508f1f61f/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp at 1730976419487Creating recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ecb70a5eda60ab5744089e2ffe59d71/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp at 1730976419489 (+2 ms)Creating recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8f449585c6b7b82f4442604100a49644/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp at 1730976419492 (+3 ms)Creating recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/db89a463448eb182960960c83f4e0b16/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp at 1730976419496 (+4 ms)3 split writer threads finished at 1730976419497 (+1 ms)Closed recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d4b381fbb1d4cd388f45774fca53b2eb/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp (wrote 1 edits, skipped 0 edits in 1 ms) at 1730976419527 (+30 ms)Rename recovered edits hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d4b381fbb1d4cd388f45774fca53b2eb/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp to hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d4b381fbb1d4cd388f45774fca53b2eb/recovered.edits/0000000000000000004 at 1730976419529 (+2 ms)Closed recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/db89a463448eb182960960c83f4e0b16/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1730976419531 (+2 ms)Closed recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7a68cc5b6ea7d9e2214ab42d65b16eba/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp (wrote 1 edits, skipped 0 edits in 1 ms) at 1730976419532 (+1 ms)Rename recovered edits hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/db89a463448eb182960960c83f4e0b16/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp to hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/db89a463448eb182960960c83f4e0b16/recovered.edits/0000000000000000004 at 1730976419534 (+2 ms)Rename recovered edits hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7a68cc5b6ea7d9e2214ab42d65b16eba/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp to hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7a68cc5b6ea7d9e2214ab42d65b16eba/recovered.edits/0000000000000000004 at 1730976419534Closed recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1701fae2bf725fdd94c16d1c6b8bc75f/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1730976419556 (+22 ms)Rename recovered edits hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1701fae2bf725fdd94c16d1c6b8bc75f/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp to hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1701fae2bf725fdd94c16d1c6b8bc75f/recovered.edits/0000000000000000004 at 1730976419557 (+1 ms)Closed recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72bf61b738a2f9b35913a192fc32815b/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp (wrote 1 edits, skipped 0 edits in 1 ms) at 1730976419585 (+28 ms)Rename recovered edits hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72bf61b738a2f9b35913a192fc32815b/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp to hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72bf61b738a2f9b35913a192fc32815b/recovered.edits/0000000000000000004 at 1730976419587 (+2 ms)Closed recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ecb70a5eda60ab5744089e2ffe59d71/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1730976419625 (+38 ms)Rename recovered edits hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ecb70a5eda60ab5744089e2ffe59d71/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp to hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ecb70a5eda60ab5744089e2ffe59d71/recovered.edits/0000000000000000004 at 1730976419629 (+4 ms)Closed recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/541874b6b96014b10121ede508f1f61f/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1730976419631 (+2 ms)Rename recovered edits hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/541874b6b96014b10121ede508f1f61f/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp to hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/541874b6b96014b10121ede508f1f61f/recovered.edits/0000000000000000004 at 1730976419633 (+2 ms)Closed recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8f449585c6b7b82f4442604100a49644/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1730976419644 (+11 ms)Rename recovered edits hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8f449585c6b7b82f4442604100a49644/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp to hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8f449585c6b7b82f4442604100a49644/recovered.edits/0000000000000000004 at 1730976419646 (+2 ms)Closed recovered edits writer path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/257c2c76c61c100547acb4a778fb1fef/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp (wrote 1 edits, skipped 0 edits in 1 ms) at 1730976419974 (+328 ms)Rename recovered edits hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/257c2c76c61c100547acb4a778fb1fef/recovered.edits/0000000000000000004-10641b2c0a69%2C41401%2C1730976230578.1730976231927.temp to hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/257c2c76c61c100547acb4a778fb1fef/recovered.edits/0000000000000000004 at 1730976419976 (+2 ms)Processed 18 edits across 9 Regions in 560 ms; skipped=9; WAL=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578-splitting/10641b2c0a69%2C41401%2C1730976230578.1730976231927, size=327.1 K, length=334915, corrupted=false, cancelled=false at 1730976419976 2024-11-07T10:46:59,976 DEBUG [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578-splitting/10641b2c0a69%2C41401%2C1730976230578.1730976231927 2024-11-07T10:46:59,976 DEBUG [RS_LOG_REPLAY_OPS-regionserver/10641b2c0a69:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-07T10:46:59,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43605 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-07T10:46:59,981 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578-splitting/10641b2c0a69%2C41401%2C1730976230578.1730976231927 to hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/oldWALs 2024-11-07T10:46:59,983 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-07T10:46:59,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 10641b2c0a69%2C41401%2C1730976230578.1730976231927, worker=10641b2c0a69,40573,1730976230476 in 756 msec 2024-11-07T10:46:59,985 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=10641b2c0a69,40573,1730976230476 2024-11-07T10:46:59,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-07T10:46:59,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 10641b2c0a69%2C41401%2C1730976230578.1730976231927, worker=10641b2c0a69,40573,1730976230476 in 764 msec 2024-11-07T10:46:59,989 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578-splitting dir is empty, no logs to split. 2024-11-07T10:46:59,989 INFO [PEWorker-1 {}] master.SplitWALManager(105): 10641b2c0a69,41401,1730976230578 WAL count=0, meta=false 2024-11-07T10:46:59,989 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 10641b2c0a69,41401,1730976230578 WAL splitting is done? wals=0, meta=false 2024-11-07T10:46:59,989 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-07T10:46:59,989 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true 2024-11-07T10:46:59,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T10:46:59,990 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T10:47:00,215 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 6 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true 2024-11-07T10:47:00,215 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T10:47:00,215 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T10:47:00,215 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T10:47:00,215 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T10:47:00,215 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T10:47:00,215 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T10:47:00,215 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T10:47:00,216 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T10:47:00,217 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T10:47:00,219 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T10:47:00,219 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T10:47:00,219 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T10:47:00,219 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T10:47:00,219 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T10:47:00,219 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T10:47:00,219 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T10:47:00,219 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T10:47:00,220 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-07T10:47:00,221 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=false; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true 2024-11-07T10:47:00,221 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476 2024-11-07T10:47:00,221 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476 2024-11-07T10:47:00,221 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T10:47:00,221 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=10641b2c0a69,40573,1730976230476 2024-11-07T10:47:00,221 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 10641b2c0a69%2C41401%2C1730976230578.1730976231927, worker=10641b2c0a69,40573,1730976230476 2024-11-07T10:47:00,221 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 10641b2c0a69%2C41401%2C1730976230578.1730976231927, worker=10641b2c0a69,40573,1730976230476 2024-11-07T10:47:00,221 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=false; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T10:47:00,222 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:00,222 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:00,222 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:00,222 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=10641b2c0a69,40573,1730976230476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:00,222 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 10641b2c0a69%2C41401%2C1730976230578.1730976231927, worker=10641b2c0a69,40573,1730976230476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:00,222 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 10641b2c0a69%2C41401%2C1730976230578.1730976231927, worker=10641b2c0a69,40573,1730976230476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:00,222 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-07T10:47:00,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=61 2024-11-07T10:47:00,223 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=10641b2c0a69,40573,1730976230476, table=hbase:meta, region=1588230740 2024-11-07T10:47:00,223 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 10641b2c0a69,40573,1730976230476, state=OPEN 2024-11-07T10:47:00,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:47:00,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:47:00,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:47:00,225 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:47:00,225 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:47:00,225 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:47:00,232 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730976450232 2024-11-07T10:47:00,232 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-07T10:47:00,239 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a68cc5b6ea7d9e2214ab42d65b16eba, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:47:00,239 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcc3491e6200809e4c9c7d529da2c4cf, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:47:00,239 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbe69e720cc4f2a4a4bc41171b6b3c32, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:47:00,239 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d4b381fbb1d4cd388f45774fca53b2eb, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:47:00,239 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f6021bbda5b8b7b00318a57f02cd77d, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:47:00,239 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddeeab6f2e44d85eb1ee916d3c5e8e9f, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:47:00,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db89a463448eb182960960c83f4e0b16, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:47:00,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3ab41a5354d389521b694050824788e, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:47:00,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f89575b73864cac8b7b164f417ef378f, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:47:00,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72bf61b738a2f9b35913a192fc32815b, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:47:00,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a7d18696a90cfed5af1fbdb2a706b65, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:47:00,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1baabb362104b46ecb78fe236391ca6f, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:47:00,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ecb70a5eda60ab5744089e2ffe59d71, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:47:00,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b830813c6b6c3ebcdb59061af10faad, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:47:00,240 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dec39ef6d94f3a519151705fce17adc4, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:47:00,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=541874b6b96014b10121ede508f1f61f, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:47:00,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64da674e3a9ec8e7458eb07874d9f204, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:47:00,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fba8e9ebafee4e67bfa1295ac44d4a5, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:47:00,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1701fae2bf725fdd94c16d1c6b8bc75f, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:47:00,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cf36f48b84f67b77887428ec6ad5b31, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:47:00,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69160268ea309aa55ece0cd8528b941d, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:47:00,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=257c2c76c61c100547acb4a778fb1fef, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:47:00,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1dd5c615432f6d3976b7d3e591d0a96a, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:47:00,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b29f5d988381aa77506fa3b92944854, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:47:00,241 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f449585c6b7b82f4442604100a49644, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:47:00,242 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c71057666e459779c2fa5a4311f4e99c, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:47:00,242 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-07T10:47:00,242 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730976480242 2024-11-07T10:47:00,242 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730976540242 2024-11-07T10:47:00,242 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-07T10:47:00,244 INFO [PEWorker-1 {}] master.MasterWalManager(331): Log dir for server 10641b2c0a69,41401,1730976230578 does not exist 2024-11-07T10:47:00,244 INFO [PEWorker-1 {}] master.SplitWALManager(105): 10641b2c0a69,41401,1730976230578 WAL count=0, meta=false 2024-11-07T10:47:00,244 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 10641b2c0a69,41401,1730976230578 WAL splitting is done? wals=0, meta=false 2024-11-07T10:47:00,245 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 10641b2c0a69,41401,1730976230578 failed, ignore...File hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/WALs/10641b2c0a69,41401,1730976230578-splitting does not exist. 2024-11-07T10:47:00,245 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-07T10:47:00,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8f449585c6b7b82f4442604100a49644, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d4b381fbb1d4cd388f45774fca53b2eb, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=72bf61b738a2f9b35913a192fc32815b, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=541874b6b96014b10121ede508f1f61f, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7a68cc5b6ea7d9e2214ab42d65b16eba, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8ecb70a5eda60ab5744089e2ffe59d71, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=db89a463448eb182960960c83f4e0b16, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1701fae2bf725fdd94c16d1c6b8bc75f, ASSIGN}, {pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=257c2c76c61c100547acb4a778fb1fef, ASSIGN}] 2024-11-07T10:47:00,247 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-07T10:47:00,247 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_FINISH, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true 2024-11-07T10:47:00,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T10:47:00,247 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_FINISH, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T10:47:00,492 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 7 exec state=pid=57, state=WAITING:SERVER_CRASH_FINISH, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true 2024-11-07T10:47:00,492 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T10:47:00,492 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T10:47:00,492 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T10:47:00,492 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T10:47:00,493 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T10:47:00,493 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T10:47:00,493 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T10:47:00,493 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T10:47:00,494 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T10:47:00,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T10:47:00,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T10:47:00,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T10:47:00,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T10:47:00,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T10:47:00,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T10:47:00,495 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T10:47:00,495 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T10:47:00,496 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-07T10:47:00,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=false; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true 2024-11-07T10:47:00,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476 2024-11-07T10:47:00,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476 2024-11-07T10:47:00,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T10:47:00,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=10641b2c0a69,40573,1730976230476 2024-11-07T10:47:00,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 10641b2c0a69%2C41401%2C1730976230578.1730976231927, worker=10641b2c0a69,40573,1730976230476 2024-11-07T10:47:00,497 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 10641b2c0a69%2C41401%2C1730976230578.1730976231927, worker=10641b2c0a69,40573,1730976230476 2024-11-07T10:47:00,497 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=false; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T10:47:00,497 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:00,497 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:00,498 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:00,498 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=10641b2c0a69,40573,1730976230476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:00,498 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 10641b2c0a69%2C41401%2C1730976230578.1730976231927, worker=10641b2c0a69,40573,1730976230476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:00,498 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 10641b2c0a69%2C41401%2C1730976230578.1730976231927, worker=10641b2c0a69,40573,1730976230476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:00,498 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-07T10:47:00,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=61 2024-11-07T10:47:00,498 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=10641b2c0a69,40573,1730976230476, table=hbase:meta, region=1588230740 2024-11-07T10:47:00,499 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 10641b2c0a69,40573,1730976230476, state=OPEN 2024-11-07T10:47:00,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:47:00,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:47:00,501 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:47:00,501 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:47:00,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:47:00,501 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:47:00,504 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730976450504 2024-11-07T10:47:00,504 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-07T10:47:00,509 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a68cc5b6ea7d9e2214ab42d65b16eba, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:47:00,509 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcc3491e6200809e4c9c7d529da2c4cf, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:47:00,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbe69e720cc4f2a4a4bc41171b6b3c32, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:47:00,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d4b381fbb1d4cd388f45774fca53b2eb, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:47:00,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f6021bbda5b8b7b00318a57f02cd77d, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:47:00,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddeeab6f2e44d85eb1ee916d3c5e8e9f, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:47:00,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db89a463448eb182960960c83f4e0b16, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:47:00,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3ab41a5354d389521b694050824788e, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:47:00,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f89575b73864cac8b7b164f417ef378f, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:47:00,510 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72bf61b738a2f9b35913a192fc32815b, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:47:00,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a7d18696a90cfed5af1fbdb2a706b65, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:47:00,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1baabb362104b46ecb78fe236391ca6f, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:47:00,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ecb70a5eda60ab5744089e2ffe59d71, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:47:00,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b830813c6b6c3ebcdb59061af10faad, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:47:00,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dec39ef6d94f3a519151705fce17adc4, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:47:00,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=541874b6b96014b10121ede508f1f61f, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:47:00,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64da674e3a9ec8e7458eb07874d9f204, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:47:00,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fba8e9ebafee4e67bfa1295ac44d4a5, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:47:00,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1701fae2bf725fdd94c16d1c6b8bc75f, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:47:00,511 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cf36f48b84f67b77887428ec6ad5b31, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:47:00,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69160268ea309aa55ece0cd8528b941d, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:47:00,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=257c2c76c61c100547acb4a778fb1fef, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:47:00,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1dd5c615432f6d3976b7d3e591d0a96a, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:47:00,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b29f5d988381aa77506fa3b92944854, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:47:00,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f449585c6b7b82f4442604100a49644, regionState=OPEN, lastHost=10641b2c0a69,41401,1730976230578, regionLocation=10641b2c0a69,41401,1730976230578, openSeqNum=2 2024-11-07T10:47:00,512 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c71057666e459779c2fa5a4311f4e99c, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:47:00,512 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-07T10:47:00,512 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730976480512 2024-11-07T10:47:00,512 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730976540512 2024-11-07T10:47:00,513 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-07T10:47:00,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8f449585c6b7b82f4442604100a49644, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d4b381fbb1d4cd388f45774fca53b2eb, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=72bf61b738a2f9b35913a192fc32815b, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=541874b6b96014b10121ede508f1f61f, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7a68cc5b6ea7d9e2214ab42d65b16eba, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8ecb70a5eda60ab5744089e2ffe59d71, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=db89a463448eb182960960c83f4e0b16, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1701fae2bf725fdd94c16d1c6b8bc75f, ASSIGN}, {pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=257c2c76c61c100547acb4a778fb1fef, ASSIGN}] 2024-11-07T10:47:00,513 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-07T10:47:00,515 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=257c2c76c61c100547acb4a778fb1fef, ASSIGN 2024-11-07T10:47:00,515 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=257c2c76c61c100547acb4a778fb1fef, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-07T10:47:00,516 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1701fae2bf725fdd94c16d1c6b8bc75f, ASSIGN 2024-11-07T10:47:00,517 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1701fae2bf725fdd94c16d1c6b8bc75f, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-07T10:47:00,518 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=db89a463448eb182960960c83f4e0b16, ASSIGN 2024-11-07T10:47:00,519 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=db89a463448eb182960960c83f4e0b16, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-07T10:47:00,519 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8ecb70a5eda60ab5744089e2ffe59d71, ASSIGN 2024-11-07T10:47:00,520 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8ecb70a5eda60ab5744089e2ffe59d71, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-07T10:47:00,521 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7a68cc5b6ea7d9e2214ab42d65b16eba, ASSIGN 2024-11-07T10:47:00,522 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7a68cc5b6ea7d9e2214ab42d65b16eba, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-07T10:47:00,522 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=541874b6b96014b10121ede508f1f61f, ASSIGN 2024-11-07T10:47:00,523 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=541874b6b96014b10121ede508f1f61f, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-07T10:47:00,524 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=72bf61b738a2f9b35913a192fc32815b, ASSIGN 2024-11-07T10:47:00,524 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=72bf61b738a2f9b35913a192fc32815b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-07T10:47:00,525 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d4b381fbb1d4cd388f45774fca53b2eb, ASSIGN 2024-11-07T10:47:00,526 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d4b381fbb1d4cd388f45774fca53b2eb, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-07T10:47:00,527 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8f449585c6b7b82f4442604100a49644, ASSIGN 2024-11-07T10:47:00,528 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8f449585c6b7b82f4442604100a49644, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-07T10:47:00,666 DEBUG [10641b2c0a69:43605 {}] balancer.BalancerClusterState(204): Hosts are {10641b2c0a69=0} racks are {/default-rack=0} 2024-11-07T10:47:00,667 DEBUG [10641b2c0a69:43605 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-07T10:47:00,667 DEBUG [10641b2c0a69:43605 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-07T10:47:00,667 DEBUG [10641b2c0a69:43605 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-07T10:47:00,667 DEBUG [10641b2c0a69:43605 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-07T10:47:00,667 INFO [10641b2c0a69:43605 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-07T10:47:00,667 INFO [10641b2c0a69:43605 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-07T10:47:00,667 DEBUG [10641b2c0a69:43605 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-07T10:47:00,668 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=257c2c76c61c100547acb4a778fb1fef, regionState=OPENING, regionLocation=10641b2c0a69,39013,1730976230620 2024-11-07T10:47:00,669 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=541874b6b96014b10121ede508f1f61f, regionState=OPENING, regionLocation=10641b2c0a69,39013,1730976230620 2024-11-07T10:47:00,669 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=72bf61b738a2f9b35913a192fc32815b, regionState=OPENING, regionLocation=10641b2c0a69,39013,1730976230620 2024-11-07T10:47:00,670 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=d4b381fbb1d4cd388f45774fca53b2eb, regionState=OPENING, regionLocation=10641b2c0a69,39013,1730976230620 2024-11-07T10:47:00,670 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=8f449585c6b7b82f4442604100a49644, regionState=OPENING, regionLocation=10641b2c0a69,40573,1730976230476 2024-11-07T10:47:00,671 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=1701fae2bf725fdd94c16d1c6b8bc75f, regionState=OPENING, regionLocation=10641b2c0a69,40573,1730976230476 2024-11-07T10:47:00,671 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=8ecb70a5eda60ab5744089e2ffe59d71, regionState=OPENING, regionLocation=10641b2c0a69,40573,1730976230476 2024-11-07T10:47:00,672 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=db89a463448eb182960960c83f4e0b16, regionState=OPENING, regionLocation=10641b2c0a69,40573,1730976230476 2024-11-07T10:47:00,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=257c2c76c61c100547acb4a778fb1fef, ASSIGN because future has completed 2024-11-07T10:47:00,672 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=7a68cc5b6ea7d9e2214ab42d65b16eba, regionState=OPENING, regionLocation=10641b2c0a69,40573,1730976230476 2024-11-07T10:47:00,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 257c2c76c61c100547acb4a778fb1fef, server=10641b2c0a69,39013,1730976230620}] 2024-11-07T10:47:00,674 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=541874b6b96014b10121ede508f1f61f, ASSIGN because future has completed 2024-11-07T10:47:00,679 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=72bf61b738a2f9b35913a192fc32815b, ASSIGN because future has completed 2024-11-07T10:47:00,683 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d4b381fbb1d4cd388f45774fca53b2eb, ASSIGN because future has completed 2024-11-07T10:47:00,698 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8f449585c6b7b82f4442604100a49644, ASSIGN because future has completed 2024-11-07T10:47:00,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f449585c6b7b82f4442604100a49644, server=10641b2c0a69,40573,1730976230476}] 2024-11-07T10:47:00,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1701fae2bf725fdd94c16d1c6b8bc75f, ASSIGN because future has completed 2024-11-07T10:47:00,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8ecb70a5eda60ab5744089e2ffe59d71, ASSIGN because future has completed 2024-11-07T10:47:00,712 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=db89a463448eb182960960c83f4e0b16, ASSIGN because future has completed 2024-11-07T10:47:00,718 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7a68cc5b6ea7d9e2214ab42d65b16eba, ASSIGN because future has completed 2024-11-07T10:47:00,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure db89a463448eb182960960c83f4e0b16, server=10641b2c0a69,40573,1730976230476}] 2024-11-07T10:47:00,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7a68cc5b6ea7d9e2214ab42d65b16eba, server=10641b2c0a69,40573,1730976230476}] 2024-11-07T10:47:00,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ecb70a5eda60ab5744089e2ffe59d71, server=10641b2c0a69,40573,1730976230476}] 2024-11-07T10:47:00,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1701fae2bf725fdd94c16d1c6b8bc75f, server=10641b2c0a69,40573,1730976230476}] 2024-11-07T10:47:00,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure d4b381fbb1d4cd388f45774fca53b2eb, server=10641b2c0a69,39013,1730976230620}] 2024-11-07T10:47:00,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 72bf61b738a2f9b35913a192fc32815b, server=10641b2c0a69,39013,1730976230620}] 2024-11-07T10:47:00,749 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 541874b6b96014b10121ede508f1f61f, server=10641b2c0a69,39013,1730976230620}] 2024-11-07T10:47:00,835 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59297, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-07T10:47:00,840 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb. 2024-11-07T10:47:00,840 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => d4b381fbb1d4cd388f45774fca53b2eb, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-07T10:47:00,841 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true d4b381fbb1d4cd388f45774fca53b2eb 2024-11-07T10:47:00,841 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:47:00,841 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for d4b381fbb1d4cd388f45774fca53b2eb 2024-11-07T10:47:00,841 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for d4b381fbb1d4cd388f45774fca53b2eb 2024-11-07T10:47:00,843 INFO [StoreOpener-d4b381fbb1d4cd388f45774fca53b2eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region d4b381fbb1d4cd388f45774fca53b2eb 2024-11-07T10:47:00,844 INFO [StoreOpener-d4b381fbb1d4cd388f45774fca53b2eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4b381fbb1d4cd388f45774fca53b2eb columnFamilyName colfamily11 2024-11-07T10:47:00,844 DEBUG [StoreOpener-d4b381fbb1d4cd388f45774fca53b2eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:47:00,844 INFO [StoreOpener-d4b381fbb1d4cd388f45774fca53b2eb-1 {}] regionserver.HStore(327): Store=d4b381fbb1d4cd388f45774fca53b2eb/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:47:00,844 INFO [StoreOpener-d4b381fbb1d4cd388f45774fca53b2eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region d4b381fbb1d4cd388f45774fca53b2eb 2024-11-07T10:47:00,845 INFO [StoreOpener-d4b381fbb1d4cd388f45774fca53b2eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4b381fbb1d4cd388f45774fca53b2eb columnFamilyName colfamily21 2024-11-07T10:47:00,845 DEBUG [StoreOpener-d4b381fbb1d4cd388f45774fca53b2eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:47:00,846 INFO [StoreOpener-d4b381fbb1d4cd388f45774fca53b2eb-1 {}] regionserver.HStore(327): Store=d4b381fbb1d4cd388f45774fca53b2eb/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:47:00,846 INFO [StoreOpener-d4b381fbb1d4cd388f45774fca53b2eb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region d4b381fbb1d4cd388f45774fca53b2eb 2024-11-07T10:47:00,847 INFO [StoreOpener-d4b381fbb1d4cd388f45774fca53b2eb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d4b381fbb1d4cd388f45774fca53b2eb columnFamilyName colfamily31 2024-11-07T10:47:00,847 DEBUG [StoreOpener-d4b381fbb1d4cd388f45774fca53b2eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:47:00,847 INFO [StoreOpener-d4b381fbb1d4cd388f45774fca53b2eb-1 {}] regionserver.HStore(327): Store=d4b381fbb1d4cd388f45774fca53b2eb/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:47:00,847 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for d4b381fbb1d4cd388f45774fca53b2eb 2024-11-07T10:47:00,848 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d4b381fbb1d4cd388f45774fca53b2eb 2024-11-07T10:47:00,850 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d4b381fbb1d4cd388f45774fca53b2eb 2024-11-07T10:47:00,851 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d4b381fbb1d4cd388f45774fca53b2eb/recovered.edits/0000000000000000004 2024-11-07T10:47:00,856 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d4b381fbb1d4cd388f45774fca53b2eb/recovered.edits/0000000000000000004 2024-11-07T10:47:00,856 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(2902): Flushing d4b381fbb1d4cd388f45774fca53b2eb 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-07T10:47:00,866 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba. 2024-11-07T10:47:00,866 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 7a68cc5b6ea7d9e2214ab42d65b16eba, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-07T10:47:00,867 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 7a68cc5b6ea7d9e2214ab42d65b16eba 2024-11-07T10:47:00,867 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:47:00,867 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 7a68cc5b6ea7d9e2214ab42d65b16eba 2024-11-07T10:47:00,867 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 7a68cc5b6ea7d9e2214ab42d65b16eba 2024-11-07T10:47:00,869 INFO [StoreOpener-7a68cc5b6ea7d9e2214ab42d65b16eba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 7a68cc5b6ea7d9e2214ab42d65b16eba 2024-11-07T10:47:00,871 INFO [StoreOpener-7a68cc5b6ea7d9e2214ab42d65b16eba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a68cc5b6ea7d9e2214ab42d65b16eba columnFamilyName colfamily11 2024-11-07T10:47:00,871 DEBUG [StoreOpener-7a68cc5b6ea7d9e2214ab42d65b16eba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:47:00,871 INFO [StoreOpener-7a68cc5b6ea7d9e2214ab42d65b16eba-1 {}] regionserver.HStore(327): Store=7a68cc5b6ea7d9e2214ab42d65b16eba/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:47:00,871 INFO [StoreOpener-7a68cc5b6ea7d9e2214ab42d65b16eba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 7a68cc5b6ea7d9e2214ab42d65b16eba 2024-11-07T10:47:00,873 INFO [StoreOpener-7a68cc5b6ea7d9e2214ab42d65b16eba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a68cc5b6ea7d9e2214ab42d65b16eba columnFamilyName colfamily21 2024-11-07T10:47:00,873 DEBUG [StoreOpener-7a68cc5b6ea7d9e2214ab42d65b16eba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:47:00,873 INFO [StoreOpener-7a68cc5b6ea7d9e2214ab42d65b16eba-1 {}] regionserver.HStore(327): Store=7a68cc5b6ea7d9e2214ab42d65b16eba/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:47:00,874 INFO [StoreOpener-7a68cc5b6ea7d9e2214ab42d65b16eba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 7a68cc5b6ea7d9e2214ab42d65b16eba 2024-11-07T10:47:00,875 INFO [StoreOpener-7a68cc5b6ea7d9e2214ab42d65b16eba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a68cc5b6ea7d9e2214ab42d65b16eba columnFamilyName colfamily31 2024-11-07T10:47:00,875 DEBUG [StoreOpener-7a68cc5b6ea7d9e2214ab42d65b16eba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:47:00,877 INFO [StoreOpener-7a68cc5b6ea7d9e2214ab42d65b16eba-1 {}] regionserver.HStore(327): Store=7a68cc5b6ea7d9e2214ab42d65b16eba/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:47:00,878 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 7a68cc5b6ea7d9e2214ab42d65b16eba 2024-11-07T10:47:00,880 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d4b381fbb1d4cd388f45774fca53b2eb/.tmp/colfamily11/6b1bc26cf3a245ecbab6a73c626152aa is 52, key is ddd/colfamily11:colfamily11/1730976236316/Put/seqid=0 2024-11-07T10:47:00,885 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7a68cc5b6ea7d9e2214ab42d65b16eba 2024-11-07T10:47:00,888 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7a68cc5b6ea7d9e2214ab42d65b16eba 2024-11-07T10:47:00,889 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7a68cc5b6ea7d9e2214ab42d65b16eba/recovered.edits/0000000000000000004 2024-11-07T10:47:00,895 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7a68cc5b6ea7d9e2214ab42d65b16eba/recovered.edits/0000000000000000004 2024-11-07T10:47:00,895 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(2902): Flushing 7a68cc5b6ea7d9e2214ab42d65b16eba 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-07T10:47:00,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741878_1054 (size=39263) 2024-11-07T10:47:00,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741878_1054 (size=39263) 2024-11-07T10:47:00,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741878_1054 (size=39263) 2024-11-07T10:47:00,915 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d4b381fbb1d4cd388f45774fca53b2eb/.tmp/colfamily11/6b1bc26cf3a245ecbab6a73c626152aa 2024-11-07T10:47:00,931 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d4b381fbb1d4cd388f45774fca53b2eb/.tmp/colfamily11/6b1bc26cf3a245ecbab6a73c626152aa as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d4b381fbb1d4cd388f45774fca53b2eb/colfamily11/6b1bc26cf3a245ecbab6a73c626152aa 2024-11-07T10:47:00,933 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7a68cc5b6ea7d9e2214ab42d65b16eba/.tmp/colfamily11/d80ab9132ca841d688dc334232c382fd is 52, key is aaa/colfamily11:colfamily11/1730976236258/Put/seqid=0 2024-11-07T10:47:00,940 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d4b381fbb1d4cd388f45774fca53b2eb/colfamily11/6b1bc26cf3a245ecbab6a73c626152aa, entries=703, sequenceid=4, filesize=38.3 K 2024-11-07T10:47:00,940 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for d4b381fbb1d4cd388f45774fca53b2eb in 84ms, sequenceid=4, compaction requested=false; wal=null 2024-11-07T10:47:00,941 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d4b381fbb1d4cd388f45774fca53b2eb/recovered.edits/0000000000000000004 2024-11-07T10:47:00,942 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for d4b381fbb1d4cd388f45774fca53b2eb 2024-11-07T10:47:00,942 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for d4b381fbb1d4cd388f45774fca53b2eb 2024-11-07T10:47:00,943 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:47:00,945 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for d4b381fbb1d4cd388f45774fca53b2eb 2024-11-07T10:47:00,953 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d4b381fbb1d4cd388f45774fca53b2eb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T10:47:00,954 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened d4b381fbb1d4cd388f45774fca53b2eb; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59155111, jitterRate=-0.11852015554904938}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:47:00,954 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d4b381fbb1d4cd388f45774fca53b2eb 2024-11-07T10:47:00,955 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for d4b381fbb1d4cd388f45774fca53b2eb: Running coprocessor pre-open hook at 1730976420841Writing region info on filesystem at 1730976420841Initializing all the Stores at 1730976420843 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976420843Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976420843Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976420843Obtaining lock to block concurrent updates at 1730976420856 (+13 ms)Preparing flush snapshotting stores in d4b381fbb1d4cd388f45774fca53b2eb at 1730976420856Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1730976420856Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb. at 1730976420856Flushing d4b381fbb1d4cd388f45774fca53b2eb/colfamily11: creating writer at 1730976420856Flushing d4b381fbb1d4cd388f45774fca53b2eb/colfamily11: appending metadata at 1730976420879 (+23 ms)Flushing d4b381fbb1d4cd388f45774fca53b2eb/colfamily11: closing flushed file at 1730976420879Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30191f16: reopening flushed file at 1730976420929 (+50 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for d4b381fbb1d4cd388f45774fca53b2eb in 84ms, sequenceid=4, compaction requested=false; wal=null at 1730976420940 (+11 ms)Cleaning up temporary data from old regions at 1730976420942 (+2 ms)Running coprocessor post-open hooks at 1730976420954 (+12 ms)Region opened successfully at 1730976420954 2024-11-07T10:47:00,956 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb., pid=79, masterSystemTime=1730976420834 2024-11-07T10:47:00,959 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=d4b381fbb1d4cd388f45774fca53b2eb, regionState=OPEN, openSeqNum=5, regionLocation=10641b2c0a69,39013,1730976230620 2024-11-07T10:47:00,959 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb. 2024-11-07T10:47:00,968 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb. 2024-11-07T10:47:00,969 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b. 2024-11-07T10:47:00,970 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => 72bf61b738a2f9b35913a192fc32815b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-07T10:47:00,970 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 72bf61b738a2f9b35913a192fc32815b 2024-11-07T10:47:00,970 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:47:00,970 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for 72bf61b738a2f9b35913a192fc32815b 2024-11-07T10:47:00,970 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for 72bf61b738a2f9b35913a192fc32815b 2024-11-07T10:47:00,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure d4b381fbb1d4cd388f45774fca53b2eb, server=10641b2c0a69,39013,1730976230620 because future has completed 2024-11-07T10:47:00,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=65 2024-11-07T10:47:00,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=65, state=SUCCESS, hasLock=false; OpenRegionProcedure d4b381fbb1d4cd388f45774fca53b2eb, server=10641b2c0a69,39013,1730976230620 in 235 msec 2024-11-07T10:47:00,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741879_1055 (size=39263) 2024-11-07T10:47:00,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741879_1055 (size=39263) 2024-11-07T10:47:00,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741879_1055 (size=39263) 2024-11-07T10:47:00,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d4b381fbb1d4cd388f45774fca53b2eb, ASSIGN in 464 msec 2024-11-07T10:47:00,980 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7a68cc5b6ea7d9e2214ab42d65b16eba/.tmp/colfamily11/d80ab9132ca841d688dc334232c382fd 2024-11-07T10:47:00,981 INFO [StoreOpener-72bf61b738a2f9b35913a192fc32815b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 72bf61b738a2f9b35913a192fc32815b 2024-11-07T10:47:00,983 INFO [StoreOpener-72bf61b738a2f9b35913a192fc32815b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72bf61b738a2f9b35913a192fc32815b columnFamilyName colfamily11 2024-11-07T10:47:00,983 DEBUG [StoreOpener-72bf61b738a2f9b35913a192fc32815b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:47:00,985 INFO [StoreOpener-72bf61b738a2f9b35913a192fc32815b-1 {}] regionserver.HStore(327): Store=72bf61b738a2f9b35913a192fc32815b/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:47:00,985 INFO [StoreOpener-72bf61b738a2f9b35913a192fc32815b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 72bf61b738a2f9b35913a192fc32815b 2024-11-07T10:47:00,987 INFO [StoreOpener-72bf61b738a2f9b35913a192fc32815b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72bf61b738a2f9b35913a192fc32815b columnFamilyName colfamily21 2024-11-07T10:47:00,987 DEBUG [StoreOpener-72bf61b738a2f9b35913a192fc32815b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:47:00,988 INFO [StoreOpener-72bf61b738a2f9b35913a192fc32815b-1 {}] regionserver.HStore(327): Store=72bf61b738a2f9b35913a192fc32815b/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:47:00,988 INFO [StoreOpener-72bf61b738a2f9b35913a192fc32815b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 72bf61b738a2f9b35913a192fc32815b 2024-11-07T10:47:00,991 INFO [StoreOpener-72bf61b738a2f9b35913a192fc32815b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 72bf61b738a2f9b35913a192fc32815b columnFamilyName colfamily31 2024-11-07T10:47:00,991 DEBUG [StoreOpener-72bf61b738a2f9b35913a192fc32815b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:47:00,997 INFO [StoreOpener-72bf61b738a2f9b35913a192fc32815b-1 {}] regionserver.HStore(327): Store=72bf61b738a2f9b35913a192fc32815b/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:47:00,998 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7a68cc5b6ea7d9e2214ab42d65b16eba/.tmp/colfamily11/d80ab9132ca841d688dc334232c382fd as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7a68cc5b6ea7d9e2214ab42d65b16eba/colfamily11/d80ab9132ca841d688dc334232c382fd 2024-11-07T10:47:00,998 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for 72bf61b738a2f9b35913a192fc32815b 2024-11-07T10:47:00,999 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72bf61b738a2f9b35913a192fc32815b 2024-11-07T10:47:01,002 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72bf61b738a2f9b35913a192fc32815b 2024-11-07T10:47:01,003 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72bf61b738a2f9b35913a192fc32815b/recovered.edits/0000000000000000004 2024-11-07T10:47:01,008 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72bf61b738a2f9b35913a192fc32815b/recovered.edits/0000000000000000004 2024-11-07T10:47:01,008 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(2902): Flushing 72bf61b738a2f9b35913a192fc32815b 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-07T10:47:01,010 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7a68cc5b6ea7d9e2214ab42d65b16eba/colfamily11/d80ab9132ca841d688dc334232c382fd, entries=703, sequenceid=4, filesize=38.3 K 2024-11-07T10:47:01,010 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 7a68cc5b6ea7d9e2214ab42d65b16eba in 115ms, sequenceid=4, compaction requested=false; wal=null 2024-11-07T10:47:01,012 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7a68cc5b6ea7d9e2214ab42d65b16eba/recovered.edits/0000000000000000004 2024-11-07T10:47:01,014 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 7a68cc5b6ea7d9e2214ab42d65b16eba 2024-11-07T10:47:01,014 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 7a68cc5b6ea7d9e2214ab42d65b16eba 2024-11-07T10:47:01,015 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:47:01,017 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 7a68cc5b6ea7d9e2214ab42d65b16eba 2024-11-07T10:47:01,024 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7a68cc5b6ea7d9e2214ab42d65b16eba/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T10:47:01,025 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 7a68cc5b6ea7d9e2214ab42d65b16eba; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61792915, jitterRate=-0.07921381294727325}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:47:01,025 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7a68cc5b6ea7d9e2214ab42d65b16eba 2024-11-07T10:47:01,025 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 7a68cc5b6ea7d9e2214ab42d65b16eba: Running coprocessor pre-open hook at 1730976420867Writing region info on filesystem at 1730976420867Initializing all the Stores at 1730976420868 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976420868Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976420868Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976420868Obtaining lock to block concurrent updates at 1730976420895 (+27 ms)Preparing flush snapshotting stores in 7a68cc5b6ea7d9e2214ab42d65b16eba at 1730976420895Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1730976420896 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba. at 1730976420896Flushing 7a68cc5b6ea7d9e2214ab42d65b16eba/colfamily11: creating writer at 1730976420896Flushing 7a68cc5b6ea7d9e2214ab42d65b16eba/colfamily11: appending metadata at 1730976420932 (+36 ms)Flushing 7a68cc5b6ea7d9e2214ab42d65b16eba/colfamily11: closing flushed file at 1730976420932Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@aa6bd28: reopening flushed file at 1730976420989 (+57 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 7a68cc5b6ea7d9e2214ab42d65b16eba in 115ms, sequenceid=4, compaction requested=false; wal=null at 1730976421011 (+22 ms)Cleaning up temporary data from old regions at 1730976421014 (+3 ms)Running coprocessor post-open hooks at 1730976421025 (+11 ms)Region opened successfully at 1730976421025 2024-11-07T10:47:01,027 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba., pid=76, masterSystemTime=1730976420862 2024-11-07T10:47:01,029 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba. 2024-11-07T10:47:01,029 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba. 2024-11-07T10:47:01,029 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16. 2024-11-07T10:47:01,030 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => db89a463448eb182960960c83f4e0b16, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-07T10:47:01,030 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true db89a463448eb182960960c83f4e0b16 2024-11-07T10:47:01,030 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=7a68cc5b6ea7d9e2214ab42d65b16eba, regionState=OPEN, openSeqNum=5, regionLocation=10641b2c0a69,40573,1730976230476 2024-11-07T10:47:01,030 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:47:01,030 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for db89a463448eb182960960c83f4e0b16 2024-11-07T10:47:01,030 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for db89a463448eb182960960c83f4e0b16 2024-11-07T10:47:01,033 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=76, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7a68cc5b6ea7d9e2214ab42d65b16eba, server=10641b2c0a69,40573,1730976230476 because future has completed 2024-11-07T10:47:01,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=68 2024-11-07T10:47:01,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 7a68cc5b6ea7d9e2214ab42d65b16eba, server=10641b2c0a69,40573,1730976230476 in 308 msec 2024-11-07T10:47:01,038 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7a68cc5b6ea7d9e2214ab42d65b16eba, ASSIGN in 523 msec 2024-11-07T10:47:01,045 INFO [StoreOpener-db89a463448eb182960960c83f4e0b16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region db89a463448eb182960960c83f4e0b16 2024-11-07T10:47:01,046 INFO [StoreOpener-db89a463448eb182960960c83f4e0b16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db89a463448eb182960960c83f4e0b16 columnFamilyName colfamily11 2024-11-07T10:47:01,046 DEBUG [StoreOpener-db89a463448eb182960960c83f4e0b16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:47:01,047 INFO [StoreOpener-db89a463448eb182960960c83f4e0b16-1 {}] regionserver.HStore(327): Store=db89a463448eb182960960c83f4e0b16/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:47:01,047 INFO [StoreOpener-db89a463448eb182960960c83f4e0b16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region db89a463448eb182960960c83f4e0b16 2024-11-07T10:47:01,048 INFO [StoreOpener-db89a463448eb182960960c83f4e0b16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db89a463448eb182960960c83f4e0b16 columnFamilyName colfamily21 2024-11-07T10:47:01,049 DEBUG [StoreOpener-db89a463448eb182960960c83f4e0b16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:47:01,050 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72bf61b738a2f9b35913a192fc32815b/.tmp/colfamily11/37e90f8ca42f4ed0aebacd39f131008d is 52, key is jjj/colfamily11:colfamily11/1730976236439/Put/seqid=0 2024-11-07T10:47:01,054 INFO [StoreOpener-db89a463448eb182960960c83f4e0b16-1 {}] regionserver.HStore(327): Store=db89a463448eb182960960c83f4e0b16/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:47:01,054 INFO [StoreOpener-db89a463448eb182960960c83f4e0b16-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region db89a463448eb182960960c83f4e0b16 2024-11-07T10:47:01,057 INFO [StoreOpener-db89a463448eb182960960c83f4e0b16-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db89a463448eb182960960c83f4e0b16 columnFamilyName colfamily31 2024-11-07T10:47:01,057 DEBUG [StoreOpener-db89a463448eb182960960c83f4e0b16-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:47:01,066 INFO [StoreOpener-db89a463448eb182960960c83f4e0b16-1 {}] regionserver.HStore(327): Store=db89a463448eb182960960c83f4e0b16/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:47:01,066 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for db89a463448eb182960960c83f4e0b16 2024-11-07T10:47:01,067 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/db89a463448eb182960960c83f4e0b16 2024-11-07T10:47:01,070 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/db89a463448eb182960960c83f4e0b16 2024-11-07T10:47:01,070 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/db89a463448eb182960960c83f4e0b16/recovered.edits/0000000000000000004 2024-11-07T10:47:01,078 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/db89a463448eb182960960c83f4e0b16/recovered.edits/0000000000000000004 2024-11-07T10:47:01,078 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(2902): Flushing db89a463448eb182960960c83f4e0b16 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-07T10:47:01,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741880_1056 (size=39263) 2024-11-07T10:47:01,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741880_1056 (size=39263) 2024-11-07T10:47:01,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741880_1056 (size=39263) 2024-11-07T10:47:01,089 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72bf61b738a2f9b35913a192fc32815b/.tmp/colfamily11/37e90f8ca42f4ed0aebacd39f131008d 2024-11-07T10:47:01,110 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72bf61b738a2f9b35913a192fc32815b/.tmp/colfamily11/37e90f8ca42f4ed0aebacd39f131008d as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72bf61b738a2f9b35913a192fc32815b/colfamily11/37e90f8ca42f4ed0aebacd39f131008d 2024-11-07T10:47:01,124 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72bf61b738a2f9b35913a192fc32815b/colfamily11/37e90f8ca42f4ed0aebacd39f131008d, entries=703, sequenceid=4, filesize=38.3 K 2024-11-07T10:47:01,124 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 72bf61b738a2f9b35913a192fc32815b in 116ms, sequenceid=4, compaction requested=false; wal=null 2024-11-07T10:47:01,125 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72bf61b738a2f9b35913a192fc32815b/recovered.edits/0000000000000000004 2024-11-07T10:47:01,127 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for 72bf61b738a2f9b35913a192fc32815b 2024-11-07T10:47:01,127 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for 72bf61b738a2f9b35913a192fc32815b 2024-11-07T10:47:01,128 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:47:01,130 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for 72bf61b738a2f9b35913a192fc32815b 2024-11-07T10:47:01,132 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/db89a463448eb182960960c83f4e0b16/.tmp/colfamily11/0b0b81e0917b4279bba1637675acd29c is 52, key is ggg/colfamily11:colfamily11/1730976236396/Put/seqid=0 2024-11-07T10:47:01,134 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72bf61b738a2f9b35913a192fc32815b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T10:47:01,135 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened 72bf61b738a2f9b35913a192fc32815b; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73658900, jitterRate=0.09760314226150513}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:47:01,135 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 72bf61b738a2f9b35913a192fc32815b 2024-11-07T10:47:01,136 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for 72bf61b738a2f9b35913a192fc32815b: Running coprocessor pre-open hook at 1730976420970Writing region info on filesystem at 1730976420970Initializing all the Stores at 1730976420974 (+4 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976420974Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976420980 (+6 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976420980Obtaining lock to block concurrent updates at 1730976421008 (+28 ms)Preparing flush snapshotting stores in 72bf61b738a2f9b35913a192fc32815b at 1730976421008Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1730976421009 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b. at 1730976421009Flushing 72bf61b738a2f9b35913a192fc32815b/colfamily11: creating writer at 1730976421009Flushing 72bf61b738a2f9b35913a192fc32815b/colfamily11: appending metadata at 1730976421049 (+40 ms)Flushing 72bf61b738a2f9b35913a192fc32815b/colfamily11: closing flushed file at 1730976421049Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@422bbebb: reopening flushed file at 1730976421107 (+58 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 72bf61b738a2f9b35913a192fc32815b in 116ms, sequenceid=4, compaction requested=false; wal=null at 1730976421124 (+17 ms)Cleaning up temporary data from old regions at 1730976421127 (+3 ms)Running coprocessor post-open hooks at 1730976421135 (+8 ms)Region opened successfully at 1730976421135 2024-11-07T10:47:01,137 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b., pid=80, masterSystemTime=1730976420834 2024-11-07T10:47:01,140 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b. 2024-11-07T10:47:01,140 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b. 2024-11-07T10:47:01,140 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef. 2024-11-07T10:47:01,140 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => 257c2c76c61c100547acb4a778fb1fef, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-07T10:47:01,140 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=72bf61b738a2f9b35913a192fc32815b, regionState=OPEN, openSeqNum=5, regionLocation=10641b2c0a69,39013,1730976230620 2024-11-07T10:47:01,141 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 257c2c76c61c100547acb4a778fb1fef 2024-11-07T10:47:01,141 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:47:01,141 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for 257c2c76c61c100547acb4a778fb1fef 2024-11-07T10:47:01,141 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for 257c2c76c61c100547acb4a778fb1fef 2024-11-07T10:47:01,144 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 72bf61b738a2f9b35913a192fc32815b, server=10641b2c0a69,39013,1730976230620 because future has completed 2024-11-07T10:47:01,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=66 2024-11-07T10:47:01,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure 72bf61b738a2f9b35913a192fc32815b, server=10641b2c0a69,39013,1730976230620 in 403 msec 2024-11-07T10:47:01,150 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=72bf61b738a2f9b35913a192fc32815b, ASSIGN in 635 msec 2024-11-07T10:47:01,160 INFO [StoreOpener-257c2c76c61c100547acb4a778fb1fef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 257c2c76c61c100547acb4a778fb1fef 2024-11-07T10:47:01,162 INFO [StoreOpener-257c2c76c61c100547acb4a778fb1fef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 257c2c76c61c100547acb4a778fb1fef columnFamilyName colfamily11 2024-11-07T10:47:01,162 DEBUG [StoreOpener-257c2c76c61c100547acb4a778fb1fef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:47:01,163 INFO [StoreOpener-257c2c76c61c100547acb4a778fb1fef-1 {}] regionserver.HStore(327): Store=257c2c76c61c100547acb4a778fb1fef/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:47:01,163 INFO [StoreOpener-257c2c76c61c100547acb4a778fb1fef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 257c2c76c61c100547acb4a778fb1fef 2024-11-07T10:47:01,164 INFO [StoreOpener-257c2c76c61c100547acb4a778fb1fef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 257c2c76c61c100547acb4a778fb1fef columnFamilyName colfamily21 2024-11-07T10:47:01,164 DEBUG [StoreOpener-257c2c76c61c100547acb4a778fb1fef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:47:01,165 INFO [StoreOpener-257c2c76c61c100547acb4a778fb1fef-1 {}] regionserver.HStore(327): Store=257c2c76c61c100547acb4a778fb1fef/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:47:01,165 INFO [StoreOpener-257c2c76c61c100547acb4a778fb1fef-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 257c2c76c61c100547acb4a778fb1fef 2024-11-07T10:47:01,166 INFO [StoreOpener-257c2c76c61c100547acb4a778fb1fef-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 257c2c76c61c100547acb4a778fb1fef columnFamilyName colfamily31 2024-11-07T10:47:01,166 DEBUG [StoreOpener-257c2c76c61c100547acb4a778fb1fef-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:47:01,166 INFO [StoreOpener-257c2c76c61c100547acb4a778fb1fef-1 {}] regionserver.HStore(327): Store=257c2c76c61c100547acb4a778fb1fef/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:47:01,166 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for 257c2c76c61c100547acb4a778fb1fef 2024-11-07T10:47:01,167 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/257c2c76c61c100547acb4a778fb1fef 2024-11-07T10:47:01,169 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/257c2c76c61c100547acb4a778fb1fef 2024-11-07T10:47:01,170 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/257c2c76c61c100547acb4a778fb1fef/recovered.edits/0000000000000000004 2024-11-07T10:47:01,177 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/257c2c76c61c100547acb4a778fb1fef/recovered.edits/0000000000000000004 2024-11-07T10:47:01,178 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(2902): Flushing 257c2c76c61c100547acb4a778fb1fef 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-07T10:47:01,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741881_1057 (size=39263) 2024-11-07T10:47:01,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741881_1057 (size=39263) 2024-11-07T10:47:01,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741881_1057 (size=39263) 2024-11-07T10:47:01,187 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/db89a463448eb182960960c83f4e0b16/.tmp/colfamily11/0b0b81e0917b4279bba1637675acd29c 2024-11-07T10:47:01,200 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/db89a463448eb182960960c83f4e0b16/.tmp/colfamily11/0b0b81e0917b4279bba1637675acd29c as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/db89a463448eb182960960c83f4e0b16/colfamily11/0b0b81e0917b4279bba1637675acd29c 2024-11-07T10:47:01,214 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/db89a463448eb182960960c83f4e0b16/colfamily11/0b0b81e0917b4279bba1637675acd29c, entries=703, sequenceid=4, filesize=38.3 K 2024-11-07T10:47:01,215 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for db89a463448eb182960960c83f4e0b16 in 137ms, sequenceid=4, compaction requested=false; wal=null 2024-11-07T10:47:01,219 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/257c2c76c61c100547acb4a778fb1fef/.tmp/colfamily11/62c7eb00a58f4b56b49850d6fd614694 is 52, key is vvv/colfamily11:colfamily11/1730976236608/Put/seqid=0 2024-11-07T10:47:01,220 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/db89a463448eb182960960c83f4e0b16/recovered.edits/0000000000000000004 2024-11-07T10:47:01,222 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for db89a463448eb182960960c83f4e0b16 2024-11-07T10:47:01,222 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for db89a463448eb182960960c83f4e0b16 2024-11-07T10:47:01,224 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:47:01,226 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for db89a463448eb182960960c83f4e0b16 2024-11-07T10:47:01,230 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/db89a463448eb182960960c83f4e0b16/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T10:47:01,232 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened db89a463448eb182960960c83f4e0b16; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59589317, jitterRate=-0.11204998195171356}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:47:01,232 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for db89a463448eb182960960c83f4e0b16 2024-11-07T10:47:01,232 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for db89a463448eb182960960c83f4e0b16: Running coprocessor pre-open hook at 1730976421030Writing region info on filesystem at 1730976421030Initializing all the Stores at 1730976421033 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976421033Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976421044 (+11 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976421044Obtaining lock to block concurrent updates at 1730976421078 (+34 ms)Preparing flush snapshotting stores in db89a463448eb182960960c83f4e0b16 at 1730976421078Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1730976421078Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16. at 1730976421078Flushing db89a463448eb182960960c83f4e0b16/colfamily11: creating writer at 1730976421078Flushing db89a463448eb182960960c83f4e0b16/colfamily11: appending metadata at 1730976421132 (+54 ms)Flushing db89a463448eb182960960c83f4e0b16/colfamily11: closing flushed file at 1730976421132Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57082784: reopening flushed file at 1730976421198 (+66 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for db89a463448eb182960960c83f4e0b16 in 137ms, sequenceid=4, compaction requested=false; wal=null at 1730976421215 (+17 ms)Cleaning up temporary data from old regions at 1730976421222 (+7 ms)Running coprocessor post-open hooks at 1730976421232 (+10 ms)Region opened successfully at 1730976421232 2024-11-07T10:47:01,234 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16., pid=75, masterSystemTime=1730976420862 2024-11-07T10:47:01,237 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16. 2024-11-07T10:47:01,237 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16. 2024-11-07T10:47:01,237 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f. 2024-11-07T10:47:01,237 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=db89a463448eb182960960c83f4e0b16, regionState=OPEN, openSeqNum=5, regionLocation=10641b2c0a69,40573,1730976230476 2024-11-07T10:47:01,237 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 1701fae2bf725fdd94c16d1c6b8bc75f, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-07T10:47:01,238 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 1701fae2bf725fdd94c16d1c6b8bc75f 2024-11-07T10:47:01,238 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:47:01,238 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 1701fae2bf725fdd94c16d1c6b8bc75f 2024-11-07T10:47:01,238 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 1701fae2bf725fdd94c16d1c6b8bc75f 2024-11-07T10:47:01,240 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure db89a463448eb182960960c83f4e0b16, server=10641b2c0a69,40573,1730976230476 because future has completed 2024-11-07T10:47:01,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=70 2024-11-07T10:47:01,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure db89a463448eb182960960c83f4e0b16, server=10641b2c0a69,40573,1730976230476 in 523 msec 2024-11-07T10:47:01,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=db89a463448eb182960960c83f4e0b16, ASSIGN in 731 msec 2024-11-07T10:47:01,253 INFO [StoreOpener-1701fae2bf725fdd94c16d1c6b8bc75f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 1701fae2bf725fdd94c16d1c6b8bc75f 2024-11-07T10:47:01,254 INFO [StoreOpener-1701fae2bf725fdd94c16d1c6b8bc75f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1701fae2bf725fdd94c16d1c6b8bc75f columnFamilyName colfamily11 2024-11-07T10:47:01,254 DEBUG [StoreOpener-1701fae2bf725fdd94c16d1c6b8bc75f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:47:01,255 INFO [StoreOpener-1701fae2bf725fdd94c16d1c6b8bc75f-1 {}] regionserver.HStore(327): Store=1701fae2bf725fdd94c16d1c6b8bc75f/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:47:01,255 INFO [StoreOpener-1701fae2bf725fdd94c16d1c6b8bc75f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 1701fae2bf725fdd94c16d1c6b8bc75f 2024-11-07T10:47:01,256 INFO [StoreOpener-1701fae2bf725fdd94c16d1c6b8bc75f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1701fae2bf725fdd94c16d1c6b8bc75f columnFamilyName colfamily21 2024-11-07T10:47:01,256 DEBUG [StoreOpener-1701fae2bf725fdd94c16d1c6b8bc75f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:47:01,257 INFO [StoreOpener-1701fae2bf725fdd94c16d1c6b8bc75f-1 {}] regionserver.HStore(327): Store=1701fae2bf725fdd94c16d1c6b8bc75f/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:47:01,257 INFO [StoreOpener-1701fae2bf725fdd94c16d1c6b8bc75f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 1701fae2bf725fdd94c16d1c6b8bc75f 2024-11-07T10:47:01,258 INFO [StoreOpener-1701fae2bf725fdd94c16d1c6b8bc75f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1701fae2bf725fdd94c16d1c6b8bc75f columnFamilyName colfamily31 2024-11-07T10:47:01,258 DEBUG [StoreOpener-1701fae2bf725fdd94c16d1c6b8bc75f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:47:01,260 INFO [StoreOpener-1701fae2bf725fdd94c16d1c6b8bc75f-1 {}] regionserver.HStore(327): Store=1701fae2bf725fdd94c16d1c6b8bc75f/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:47:01,260 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 1701fae2bf725fdd94c16d1c6b8bc75f 2024-11-07T10:47:01,261 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1701fae2bf725fdd94c16d1c6b8bc75f 2024-11-07T10:47:01,265 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1701fae2bf725fdd94c16d1c6b8bc75f 2024-11-07T10:47:01,265 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1701fae2bf725fdd94c16d1c6b8bc75f/recovered.edits/0000000000000000004 2024-11-07T10:47:01,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741882_1058 (size=39263) 2024-11-07T10:47:01,271 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1701fae2bf725fdd94c16d1c6b8bc75f/recovered.edits/0000000000000000004 2024-11-07T10:47:01,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741882_1058 (size=39263) 2024-11-07T10:47:01,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741882_1058 (size=39263) 2024-11-07T10:47:01,272 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(2902): Flushing 1701fae2bf725fdd94c16d1c6b8bc75f 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-07T10:47:01,273 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/257c2c76c61c100547acb4a778fb1fef/.tmp/colfamily11/62c7eb00a58f4b56b49850d6fd614694 2024-11-07T10:47:01,283 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/257c2c76c61c100547acb4a778fb1fef/.tmp/colfamily11/62c7eb00a58f4b56b49850d6fd614694 as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/257c2c76c61c100547acb4a778fb1fef/colfamily11/62c7eb00a58f4b56b49850d6fd614694 2024-11-07T10:47:01,290 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/257c2c76c61c100547acb4a778fb1fef/colfamily11/62c7eb00a58f4b56b49850d6fd614694, entries=703, sequenceid=4, filesize=38.3 K 2024-11-07T10:47:01,290 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 257c2c76c61c100547acb4a778fb1fef in 113ms, sequenceid=4, compaction requested=false; wal=null 2024-11-07T10:47:01,291 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/257c2c76c61c100547acb4a778fb1fef/recovered.edits/0000000000000000004 2024-11-07T10:47:01,293 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for 257c2c76c61c100547acb4a778fb1fef 2024-11-07T10:47:01,293 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for 257c2c76c61c100547acb4a778fb1fef 2024-11-07T10:47:01,294 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:47:01,295 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for 257c2c76c61c100547acb4a778fb1fef 2024-11-07T10:47:01,298 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/257c2c76c61c100547acb4a778fb1fef/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T10:47:01,303 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1701fae2bf725fdd94c16d1c6b8bc75f/.tmp/colfamily11/de6d3b5faac64c4987196ac4e186c845 is 52, key is sss/colfamily11:colfamily11/1730976236565/Put/seqid=0 2024-11-07T10:47:01,305 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened 257c2c76c61c100547acb4a778fb1fef; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72606933, jitterRate=0.08192761242389679}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:47:01,305 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 257c2c76c61c100547acb4a778fb1fef 2024-11-07T10:47:01,305 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for 257c2c76c61c100547acb4a778fb1fef: Running coprocessor pre-open hook at 1730976421141Writing region info on filesystem at 1730976421141Initializing all the Stores at 1730976421143 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976421143Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976421160 (+17 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976421160Obtaining lock to block concurrent updates at 1730976421178 (+18 ms)Preparing flush snapshotting stores in 257c2c76c61c100547acb4a778fb1fef at 1730976421178Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1730976421178Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef. at 1730976421178Flushing 257c2c76c61c100547acb4a778fb1fef/colfamily11: creating writer at 1730976421178Flushing 257c2c76c61c100547acb4a778fb1fef/colfamily11: appending metadata at 1730976421218 (+40 ms)Flushing 257c2c76c61c100547acb4a778fb1fef/colfamily11: closing flushed file at 1730976421218Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@645df2ef: reopening flushed file at 1730976421282 (+64 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 257c2c76c61c100547acb4a778fb1fef in 113ms, sequenceid=4, compaction requested=false; wal=null at 1730976421291 (+9 ms)Cleaning up temporary data from old regions at 1730976421293 (+2 ms)Running coprocessor post-open hooks at 1730976421305 (+12 ms)Region opened successfully at 1730976421305 2024-11-07T10:47:01,306 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef., pid=73, masterSystemTime=1730976420834 2024-11-07T10:47:01,310 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef. 2024-11-07T10:47:01,310 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef. 2024-11-07T10:47:01,310 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f. 2024-11-07T10:47:01,310 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=257c2c76c61c100547acb4a778fb1fef, regionState=OPEN, openSeqNum=5, regionLocation=10641b2c0a69,39013,1730976230620 2024-11-07T10:47:01,310 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => 541874b6b96014b10121ede508f1f61f, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-07T10:47:01,310 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 541874b6b96014b10121ede508f1f61f 2024-11-07T10:47:01,310 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:47:01,311 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for 541874b6b96014b10121ede508f1f61f 2024-11-07T10:47:01,311 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for 541874b6b96014b10121ede508f1f61f 2024-11-07T10:47:01,312 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 257c2c76c61c100547acb4a778fb1fef, server=10641b2c0a69,39013,1730976230620 because future has completed 2024-11-07T10:47:01,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-07T10:47:01,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; OpenRegionProcedure 257c2c76c61c100547acb4a778fb1fef, server=10641b2c0a69,39013,1730976230620 in 639 msec 2024-11-07T10:47:01,321 INFO [StoreOpener-541874b6b96014b10121ede508f1f61f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 541874b6b96014b10121ede508f1f61f 2024-11-07T10:47:01,321 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=257c2c76c61c100547acb4a778fb1fef, ASSIGN in 805 msec 2024-11-07T10:47:01,322 INFO [StoreOpener-541874b6b96014b10121ede508f1f61f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 541874b6b96014b10121ede508f1f61f columnFamilyName colfamily11 2024-11-07T10:47:01,322 DEBUG [StoreOpener-541874b6b96014b10121ede508f1f61f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:47:01,323 INFO [StoreOpener-541874b6b96014b10121ede508f1f61f-1 {}] regionserver.HStore(327): Store=541874b6b96014b10121ede508f1f61f/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:47:01,323 INFO [StoreOpener-541874b6b96014b10121ede508f1f61f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 541874b6b96014b10121ede508f1f61f 2024-11-07T10:47:01,324 INFO [StoreOpener-541874b6b96014b10121ede508f1f61f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 541874b6b96014b10121ede508f1f61f columnFamilyName colfamily21 2024-11-07T10:47:01,324 DEBUG [StoreOpener-541874b6b96014b10121ede508f1f61f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:47:01,324 INFO [StoreOpener-541874b6b96014b10121ede508f1f61f-1 {}] regionserver.HStore(327): Store=541874b6b96014b10121ede508f1f61f/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:47:01,324 INFO [StoreOpener-541874b6b96014b10121ede508f1f61f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 541874b6b96014b10121ede508f1f61f 2024-11-07T10:47:01,326 INFO [StoreOpener-541874b6b96014b10121ede508f1f61f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 541874b6b96014b10121ede508f1f61f columnFamilyName colfamily31 2024-11-07T10:47:01,326 DEBUG [StoreOpener-541874b6b96014b10121ede508f1f61f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:47:01,326 INFO [StoreOpener-541874b6b96014b10121ede508f1f61f-1 {}] regionserver.HStore(327): Store=541874b6b96014b10121ede508f1f61f/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:47:01,326 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for 541874b6b96014b10121ede508f1f61f 2024-11-07T10:47:01,327 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/541874b6b96014b10121ede508f1f61f 2024-11-07T10:47:01,329 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/541874b6b96014b10121ede508f1f61f 2024-11-07T10:47:01,330 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/541874b6b96014b10121ede508f1f61f/recovered.edits/0000000000000000004 2024-11-07T10:47:01,337 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/541874b6b96014b10121ede508f1f61f/recovered.edits/0000000000000000004 2024-11-07T10:47:01,337 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(2902): Flushing 541874b6b96014b10121ede508f1f61f 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-07T10:47:01,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741883_1059 (size=39263) 2024-11-07T10:47:01,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741883_1059 (size=39263) 2024-11-07T10:47:01,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741883_1059 (size=39263) 2024-11-07T10:47:01,364 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1701fae2bf725fdd94c16d1c6b8bc75f/.tmp/colfamily11/de6d3b5faac64c4987196ac4e186c845 2024-11-07T10:47:01,374 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1701fae2bf725fdd94c16d1c6b8bc75f/.tmp/colfamily11/de6d3b5faac64c4987196ac4e186c845 as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1701fae2bf725fdd94c16d1c6b8bc75f/colfamily11/de6d3b5faac64c4987196ac4e186c845 2024-11-07T10:47:01,381 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/541874b6b96014b10121ede508f1f61f/.tmp/colfamily11/1a12394b1a184aaa8360a37759925ee8 is 52, key is ppp/colfamily11:colfamily11/1730976236524/Put/seqid=0 2024-11-07T10:47:01,382 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1701fae2bf725fdd94c16d1c6b8bc75f/colfamily11/de6d3b5faac64c4987196ac4e186c845, entries=703, sequenceid=4, filesize=38.3 K 2024-11-07T10:47:01,382 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 1701fae2bf725fdd94c16d1c6b8bc75f in 111ms, sequenceid=4, compaction requested=false; wal=null 2024-11-07T10:47:01,384 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1701fae2bf725fdd94c16d1c6b8bc75f/recovered.edits/0000000000000000004 2024-11-07T10:47:01,385 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 1701fae2bf725fdd94c16d1c6b8bc75f 2024-11-07T10:47:01,385 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 1701fae2bf725fdd94c16d1c6b8bc75f 2024-11-07T10:47:01,386 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:47:01,388 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 1701fae2bf725fdd94c16d1c6b8bc75f 2024-11-07T10:47:01,401 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1701fae2bf725fdd94c16d1c6b8bc75f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T10:47:01,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741884_1060 (size=39263) 2024-11-07T10:47:01,402 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 1701fae2bf725fdd94c16d1c6b8bc75f; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59622716, jitterRate=-0.11155229806900024}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:47:01,402 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1701fae2bf725fdd94c16d1c6b8bc75f 2024-11-07T10:47:01,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741884_1060 (size=39263) 2024-11-07T10:47:01,402 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 1701fae2bf725fdd94c16d1c6b8bc75f: Running coprocessor pre-open hook at 1730976421239Writing region info on filesystem at 1730976421239Initializing all the Stores at 1730976421240 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976421241 (+1 ms)Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976421252 (+11 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976421252Obtaining lock to block concurrent updates at 1730976421272 (+20 ms)Preparing flush snapshotting stores in 1701fae2bf725fdd94c16d1c6b8bc75f at 1730976421272Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1730976421273 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f. at 1730976421273Flushing 1701fae2bf725fdd94c16d1c6b8bc75f/colfamily11: creating writer at 1730976421273Flushing 1701fae2bf725fdd94c16d1c6b8bc75f/colfamily11: appending metadata at 1730976421302 (+29 ms)Flushing 1701fae2bf725fdd94c16d1c6b8bc75f/colfamily11: closing flushed file at 1730976421302Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e417dfd: reopening flushed file at 1730976421373 (+71 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 1701fae2bf725fdd94c16d1c6b8bc75f in 111ms, sequenceid=4, compaction requested=false; wal=null at 1730976421382 (+9 ms)Cleaning up temporary data from old regions at 1730976421385 (+3 ms)Running coprocessor post-open hooks at 1730976421402 (+17 ms)Region opened successfully at 1730976421402 2024-11-07T10:47:01,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741884_1060 (size=39263) 2024-11-07T10:47:01,404 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f., pid=78, masterSystemTime=1730976420862 2024-11-07T10:47:01,405 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/541874b6b96014b10121ede508f1f61f/.tmp/colfamily11/1a12394b1a184aaa8360a37759925ee8 2024-11-07T10:47:01,408 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=1701fae2bf725fdd94c16d1c6b8bc75f, regionState=OPEN, openSeqNum=5, regionLocation=10641b2c0a69,40573,1730976230476 2024-11-07T10:47:01,409 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f. 2024-11-07T10:47:01,409 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f. 2024-11-07T10:47:01,409 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644. 2024-11-07T10:47:01,409 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => 8f449585c6b7b82f4442604100a49644, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-07T10:47:01,410 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 8f449585c6b7b82f4442604100a49644 2024-11-07T10:47:01,410 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:47:01,410 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for 8f449585c6b7b82f4442604100a49644 2024-11-07T10:47:01,410 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for 8f449585c6b7b82f4442604100a49644 2024-11-07T10:47:01,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1701fae2bf725fdd94c16d1c6b8bc75f, server=10641b2c0a69,40573,1730976230476 because future has completed 2024-11-07T10:47:01,413 INFO [StoreOpener-8f449585c6b7b82f4442604100a49644-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 8f449585c6b7b82f4442604100a49644 2024-11-07T10:47:01,414 INFO [StoreOpener-8f449585c6b7b82f4442604100a49644-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f449585c6b7b82f4442604100a49644 columnFamilyName colfamily11 2024-11-07T10:47:01,414 DEBUG [StoreOpener-8f449585c6b7b82f4442604100a49644-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:47:01,415 INFO [StoreOpener-8f449585c6b7b82f4442604100a49644-1 {}] regionserver.HStore(327): Store=8f449585c6b7b82f4442604100a49644/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:47:01,416 INFO [StoreOpener-8f449585c6b7b82f4442604100a49644-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 8f449585c6b7b82f4442604100a49644 2024-11-07T10:47:01,416 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/541874b6b96014b10121ede508f1f61f/.tmp/colfamily11/1a12394b1a184aaa8360a37759925ee8 as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/541874b6b96014b10121ede508f1f61f/colfamily11/1a12394b1a184aaa8360a37759925ee8 2024-11-07T10:47:01,417 INFO [StoreOpener-8f449585c6b7b82f4442604100a49644-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f449585c6b7b82f4442604100a49644 columnFamilyName colfamily21 2024-11-07T10:47:01,417 DEBUG [StoreOpener-8f449585c6b7b82f4442604100a49644-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:47:01,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=71 2024-11-07T10:47:01,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=71, state=SUCCESS, hasLock=false; OpenRegionProcedure 1701fae2bf725fdd94c16d1c6b8bc75f, server=10641b2c0a69,40573,1730976230476 in 679 msec 2024-11-07T10:47:01,419 INFO [StoreOpener-8f449585c6b7b82f4442604100a49644-1 {}] regionserver.HStore(327): Store=8f449585c6b7b82f4442604100a49644/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:47:01,419 INFO [StoreOpener-8f449585c6b7b82f4442604100a49644-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 8f449585c6b7b82f4442604100a49644 2024-11-07T10:47:01,420 INFO [StoreOpener-8f449585c6b7b82f4442604100a49644-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f449585c6b7b82f4442604100a49644 columnFamilyName colfamily31 2024-11-07T10:47:01,420 DEBUG [StoreOpener-8f449585c6b7b82f4442604100a49644-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:47:01,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1701fae2bf725fdd94c16d1c6b8bc75f, ASSIGN in 905 msec 2024-11-07T10:47:01,420 INFO [StoreOpener-8f449585c6b7b82f4442604100a49644-1 {}] regionserver.HStore(327): Store=8f449585c6b7b82f4442604100a49644/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:47:01,421 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for 8f449585c6b7b82f4442604100a49644 2024-11-07T10:47:01,421 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8f449585c6b7b82f4442604100a49644 2024-11-07T10:47:01,424 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8f449585c6b7b82f4442604100a49644 2024-11-07T10:47:01,425 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8f449585c6b7b82f4442604100a49644/recovered.edits/0000000000000000004 2024-11-07T10:47:01,426 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/541874b6b96014b10121ede508f1f61f/colfamily11/1a12394b1a184aaa8360a37759925ee8, entries=703, sequenceid=4, filesize=38.3 K 2024-11-07T10:47:01,426 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 541874b6b96014b10121ede508f1f61f in 89ms, sequenceid=4, compaction requested=false; wal=null 2024-11-07T10:47:01,428 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/541874b6b96014b10121ede508f1f61f/recovered.edits/0000000000000000004 2024-11-07T10:47:01,429 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for 541874b6b96014b10121ede508f1f61f 2024-11-07T10:47:01,429 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for 541874b6b96014b10121ede508f1f61f 2024-11-07T10:47:01,430 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:47:01,431 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8f449585c6b7b82f4442604100a49644/recovered.edits/0000000000000000004 2024-11-07T10:47:01,431 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(2902): Flushing 8f449585c6b7b82f4442604100a49644 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-07T10:47:01,433 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for 541874b6b96014b10121ede508f1f61f 2024-11-07T10:47:01,445 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/541874b6b96014b10121ede508f1f61f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T10:47:01,446 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened 541874b6b96014b10121ede508f1f61f; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68187131, jitterRate=0.01606743037700653}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:47:01,446 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 541874b6b96014b10121ede508f1f61f 2024-11-07T10:47:01,447 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for 541874b6b96014b10121ede508f1f61f: Running coprocessor pre-open hook at 1730976421311Writing region info on filesystem at 1730976421311Initializing all the Stores at 1730976421313 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976421313Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976421320 (+7 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976421320Obtaining lock to block concurrent updates at 1730976421337 (+17 ms)Preparing flush snapshotting stores in 541874b6b96014b10121ede508f1f61f at 1730976421337Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1730976421337Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f. at 1730976421337Flushing 541874b6b96014b10121ede508f1f61f/colfamily11: creating writer at 1730976421337Flushing 541874b6b96014b10121ede508f1f61f/colfamily11: appending metadata at 1730976421380 (+43 ms)Flushing 541874b6b96014b10121ede508f1f61f/colfamily11: closing flushed file at 1730976421380Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5799ffe7: reopening flushed file at 1730976421414 (+34 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 541874b6b96014b10121ede508f1f61f in 89ms, sequenceid=4, compaction requested=false; wal=null at 1730976421426 (+12 ms)Cleaning up temporary data from old regions at 1730976421429 (+3 ms)Running coprocessor post-open hooks at 1730976421446 (+17 ms)Region opened successfully at 1730976421446 2024-11-07T10:47:01,447 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f., pid=81, masterSystemTime=1730976420834 2024-11-07T10:47:01,450 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f. 2024-11-07T10:47:01,450 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f. 2024-11-07T10:47:01,451 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=541874b6b96014b10121ede508f1f61f, regionState=OPEN, openSeqNum=5, regionLocation=10641b2c0a69,39013,1730976230620 2024-11-07T10:47:01,453 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 541874b6b96014b10121ede508f1f61f, server=10641b2c0a69,39013,1730976230620 because future has completed 2024-11-07T10:47:01,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=67 2024-11-07T10:47:01,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure 541874b6b96014b10121ede508f1f61f, server=10641b2c0a69,39013,1730976230620 in 706 msec 2024-11-07T10:47:01,458 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8f449585c6b7b82f4442604100a49644/.tmp/colfamily11/293e8a79dc57409683f2f01cf00cb357 is 52, key is yyy/colfamily11:colfamily11/1730976236655/Put/seqid=0 2024-11-07T10:47:01,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=541874b6b96014b10121ede508f1f61f, ASSIGN in 943 msec 2024-11-07T10:47:01,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741885_1061 (size=39263) 2024-11-07T10:47:01,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741885_1061 (size=39263) 2024-11-07T10:47:01,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741885_1061 (size=39263) 2024-11-07T10:47:01,470 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8f449585c6b7b82f4442604100a49644/.tmp/colfamily11/293e8a79dc57409683f2f01cf00cb357 2024-11-07T10:47:01,478 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8f449585c6b7b82f4442604100a49644/.tmp/colfamily11/293e8a79dc57409683f2f01cf00cb357 as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8f449585c6b7b82f4442604100a49644/colfamily11/293e8a79dc57409683f2f01cf00cb357 2024-11-07T10:47:01,484 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8f449585c6b7b82f4442604100a49644/colfamily11/293e8a79dc57409683f2f01cf00cb357, entries=703, sequenceid=4, filesize=38.3 K 2024-11-07T10:47:01,484 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 8f449585c6b7b82f4442604100a49644 in 53ms, sequenceid=4, compaction requested=false; wal=null 2024-11-07T10:47:01,485 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8f449585c6b7b82f4442604100a49644/recovered.edits/0000000000000000004 2024-11-07T10:47:01,486 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for 8f449585c6b7b82f4442604100a49644 2024-11-07T10:47:01,486 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for 8f449585c6b7b82f4442604100a49644 2024-11-07T10:47:01,487 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:47:01,488 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for 8f449585c6b7b82f4442604100a49644 2024-11-07T10:47:01,490 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8f449585c6b7b82f4442604100a49644/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T10:47:01,491 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened 8f449585c6b7b82f4442604100a49644; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67839188, jitterRate=0.010882675647735596}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:47:01,491 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8f449585c6b7b82f4442604100a49644 2024-11-07T10:47:01,491 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for 8f449585c6b7b82f4442604100a49644: Running coprocessor pre-open hook at 1730976421410Writing region info on filesystem at 1730976421410Initializing all the Stores at 1730976421412 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976421412Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976421413 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976421413Obtaining lock to block concurrent updates at 1730976421431 (+18 ms)Preparing flush snapshotting stores in 8f449585c6b7b82f4442604100a49644 at 1730976421431Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1730976421431Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644. at 1730976421431Flushing 8f449585c6b7b82f4442604100a49644/colfamily11: creating writer at 1730976421431Flushing 8f449585c6b7b82f4442604100a49644/colfamily11: appending metadata at 1730976421457 (+26 ms)Flushing 8f449585c6b7b82f4442604100a49644/colfamily11: closing flushed file at 1730976421457Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50b3d93d: reopening flushed file at 1730976421477 (+20 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 8f449585c6b7b82f4442604100a49644 in 53ms, sequenceid=4, compaction requested=false; wal=null at 1730976421484 (+7 ms)Cleaning up temporary data from old regions at 1730976421486 (+2 ms)Running coprocessor post-open hooks at 1730976421491 (+5 ms)Region opened successfully at 1730976421491 2024-11-07T10:47:01,492 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644., pid=74, masterSystemTime=1730976420862 2024-11-07T10:47:01,494 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644. 2024-11-07T10:47:01,494 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644. 2024-11-07T10:47:01,494 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71. 2024-11-07T10:47:01,494 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 8ecb70a5eda60ab5744089e2ffe59d71, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-07T10:47:01,495 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 8ecb70a5eda60ab5744089e2ffe59d71 2024-11-07T10:47:01,495 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-07T10:47:01,495 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 8ecb70a5eda60ab5744089e2ffe59d71 2024-11-07T10:47:01,495 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 8ecb70a5eda60ab5744089e2ffe59d71 2024-11-07T10:47:01,495 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=8f449585c6b7b82f4442604100a49644, regionState=OPEN, openSeqNum=5, regionLocation=10641b2c0a69,40573,1730976230476 2024-11-07T10:47:01,497 INFO [StoreOpener-8ecb70a5eda60ab5744089e2ffe59d71-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 8ecb70a5eda60ab5744089e2ffe59d71 2024-11-07T10:47:01,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f449585c6b7b82f4442604100a49644, server=10641b2c0a69,40573,1730976230476 because future has completed 2024-11-07T10:47:01,498 INFO [StoreOpener-8ecb70a5eda60ab5744089e2ffe59d71-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ecb70a5eda60ab5744089e2ffe59d71 columnFamilyName colfamily11 2024-11-07T10:47:01,498 DEBUG [StoreOpener-8ecb70a5eda60ab5744089e2ffe59d71-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:47:01,498 INFO [StoreOpener-8ecb70a5eda60ab5744089e2ffe59d71-1 {}] regionserver.HStore(327): Store=8ecb70a5eda60ab5744089e2ffe59d71/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:47:01,499 INFO [StoreOpener-8ecb70a5eda60ab5744089e2ffe59d71-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 8ecb70a5eda60ab5744089e2ffe59d71 2024-11-07T10:47:01,499 INFO [StoreOpener-8ecb70a5eda60ab5744089e2ffe59d71-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ecb70a5eda60ab5744089e2ffe59d71 columnFamilyName colfamily21 2024-11-07T10:47:01,500 DEBUG [StoreOpener-8ecb70a5eda60ab5744089e2ffe59d71-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:47:01,500 INFO [StoreOpener-8ecb70a5eda60ab5744089e2ffe59d71-1 {}] regionserver.HStore(327): Store=8ecb70a5eda60ab5744089e2ffe59d71/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:47:01,500 INFO [StoreOpener-8ecb70a5eda60ab5744089e2ffe59d71-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 8ecb70a5eda60ab5744089e2ffe59d71 2024-11-07T10:47:01,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=64 2024-11-07T10:47:01,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 8f449585c6b7b82f4442604100a49644, server=10641b2c0a69,40573,1730976230476 in 798 msec 2024-11-07T10:47:01,501 INFO [StoreOpener-8ecb70a5eda60ab5744089e2ffe59d71-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ecb70a5eda60ab5744089e2ffe59d71 columnFamilyName colfamily31 2024-11-07T10:47:01,501 DEBUG [StoreOpener-8ecb70a5eda60ab5744089e2ffe59d71-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-07T10:47:01,502 INFO [StoreOpener-8ecb70a5eda60ab5744089e2ffe59d71-1 {}] regionserver.HStore(327): Store=8ecb70a5eda60ab5744089e2ffe59d71/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-07T10:47:01,502 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 8ecb70a5eda60ab5744089e2ffe59d71 2024-11-07T10:47:01,503 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ecb70a5eda60ab5744089e2ffe59d71 2024-11-07T10:47:01,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8f449585c6b7b82f4442604100a49644, ASSIGN in 988 msec 2024-11-07T10:47:01,504 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ecb70a5eda60ab5744089e2ffe59d71 2024-11-07T10:47:01,505 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ecb70a5eda60ab5744089e2ffe59d71/recovered.edits/0000000000000000004 2024-11-07T10:47:01,509 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ecb70a5eda60ab5744089e2ffe59d71/recovered.edits/0000000000000000004 2024-11-07T10:47:01,509 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(2902): Flushing 8ecb70a5eda60ab5744089e2ffe59d71 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-07T10:47:01,531 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ecb70a5eda60ab5744089e2ffe59d71/.tmp/colfamily11/f2873bf8cd83462b8e870898d447e792 is 52, key is mmm/colfamily11:colfamily11/1730976236496/Put/seqid=0 2024-11-07T10:47:01,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741886_1062 (size=39263) 2024-11-07T10:47:01,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741886_1062 (size=39263) 2024-11-07T10:47:01,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741886_1062 (size=39263) 2024-11-07T10:47:01,552 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ecb70a5eda60ab5744089e2ffe59d71/.tmp/colfamily11/f2873bf8cd83462b8e870898d447e792 2024-11-07T10:47:01,560 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ecb70a5eda60ab5744089e2ffe59d71/.tmp/colfamily11/f2873bf8cd83462b8e870898d447e792 as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ecb70a5eda60ab5744089e2ffe59d71/colfamily11/f2873bf8cd83462b8e870898d447e792 2024-11-07T10:47:01,566 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ecb70a5eda60ab5744089e2ffe59d71/colfamily11/f2873bf8cd83462b8e870898d447e792, entries=703, sequenceid=4, filesize=38.3 K 2024-11-07T10:47:01,567 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 8ecb70a5eda60ab5744089e2ffe59d71 in 57ms, sequenceid=4, compaction requested=false; wal=null 2024-11-07T10:47:01,567 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ecb70a5eda60ab5744089e2ffe59d71/recovered.edits/0000000000000000004 2024-11-07T10:47:01,569 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 8ecb70a5eda60ab5744089e2ffe59d71 2024-11-07T10:47:01,569 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 8ecb70a5eda60ab5744089e2ffe59d71 2024-11-07T10:47:01,570 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-07T10:47:01,571 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 8ecb70a5eda60ab5744089e2ffe59d71 2024-11-07T10:47:01,588 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ecb70a5eda60ab5744089e2ffe59d71/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-07T10:47:01,589 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 8ecb70a5eda60ab5744089e2ffe59d71; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59363236, jitterRate=-0.11541885137557983}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-07T10:47:01,589 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8ecb70a5eda60ab5744089e2ffe59d71 2024-11-07T10:47:01,589 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 8ecb70a5eda60ab5744089e2ffe59d71: Running coprocessor pre-open hook at 1730976421495Writing region info on filesystem at 1730976421495Initializing all the Stores at 1730976421496 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976421496Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976421497 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1730976421497Obtaining lock to block concurrent updates at 1730976421510 (+13 ms)Preparing flush snapshotting stores in 8ecb70a5eda60ab5744089e2ffe59d71 at 1730976421510Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1730976421510Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71. at 1730976421510Flushing 8ecb70a5eda60ab5744089e2ffe59d71/colfamily11: creating writer at 1730976421510Flushing 8ecb70a5eda60ab5744089e2ffe59d71/colfamily11: appending metadata at 1730976421530 (+20 ms)Flushing 8ecb70a5eda60ab5744089e2ffe59d71/colfamily11: closing flushed file at 1730976421530Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@133e50f9: reopening flushed file at 1730976421559 (+29 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 8ecb70a5eda60ab5744089e2ffe59d71 in 57ms, sequenceid=4, compaction requested=false; wal=null at 1730976421567 (+8 ms)Cleaning up temporary data from old regions at 1730976421569 (+2 ms)Running coprocessor post-open hooks at 1730976421589 (+20 ms)Region opened successfully at 1730976421589 2024-11-07T10:47:01,590 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71., pid=77, masterSystemTime=1730976420862 2024-11-07T10:47:01,592 DEBUG [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71. 2024-11-07T10:47:01,592 INFO [RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71. 2024-11-07T10:47:01,593 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=8ecb70a5eda60ab5744089e2ffe59d71, regionState=OPEN, openSeqNum=5, regionLocation=10641b2c0a69,40573,1730976230476 2024-11-07T10:47:01,595 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ecb70a5eda60ab5744089e2ffe59d71, server=10641b2c0a69,40573,1730976230476 because future has completed 2024-11-07T10:47:01,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=69 2024-11-07T10:47:01,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure 8ecb70a5eda60ab5744089e2ffe59d71, server=10641b2c0a69,40573,1730976230476 in 865 msec 2024-11-07T10:47:01,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=57 2024-11-07T10:47:01,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8ecb70a5eda60ab5744089e2ffe59d71, ASSIGN in 1.0860 sec 2024-11-07T10:47:01,601 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 10641b2c0a69,41401,1730976230578 after splitting done 2024-11-07T10:47:01,601 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-07T10:47:01,601 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=SUCCESS, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true 2024-11-07T10:47:01,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T10:47:01,602 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=SUCCESS, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-07T10:47:01,763 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 8 exec state=pid=57, state=SUCCESS, hasLock=true; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true 2024-11-07T10:47:01,763 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-07T10:47:01,763 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T10:47:01,763 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T10:47:01,763 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T10:47:01,764 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-07T10:47:01,764 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-07T10:47:01,764 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-07T10:47:01,764 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-07T10:47:01,765 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-07T10:47:01,767 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-07T10:47:01,767 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T10:47:01,767 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T10:47:01,767 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T10:47:01,767 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T10:47:01,767 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T10:47:01,767 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T10:47:01,767 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T10:47:01,767 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T10:47:01,767 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T10:47:01,767 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T10:47:01,767 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T10:47:01,767 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T10:47:01,767 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T10:47:01,768 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T10:47:01,768 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T10:47:01,768 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T10:47:01,768 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T10:47:01,768 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T10:47:01,768 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T10:47:01,768 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T10:47:01,768 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-07T10:47:01,768 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-07T10:47:01,768 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-07T10:47:01,768 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-07T10:47:01,768 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-07T10:47:01,769 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-07T10:47:01,772 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_FINISH, hasLock=false; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true 2024-11-07T10:47:01,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476 2024-11-07T10:47:01,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476 2024-11-07T10:47:01,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-07T10:47:01,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=10641b2c0a69,40573,1730976230476 2024-11-07T10:47:01,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 10641b2c0a69%2C41401%2C1730976230578.1730976231927, worker=10641b2c0a69,40573,1730976230476 2024-11-07T10:47:01,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 10641b2c0a69%2C41401%2C1730976230578.1730976231927, worker=10641b2c0a69,40573,1730976230476 2024-11-07T10:47:01,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8f449585c6b7b82f4442604100a49644, ASSIGN 2024-11-07T10:47:01,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d4b381fbb1d4cd388f45774fca53b2eb, ASSIGN 2024-11-07T10:47:01,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=72bf61b738a2f9b35913a192fc32815b, ASSIGN 2024-11-07T10:47:01,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=541874b6b96014b10121ede508f1f61f, ASSIGN 2024-11-07T10:47:01,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7a68cc5b6ea7d9e2214ab42d65b16eba, ASSIGN 2024-11-07T10:47:01,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8ecb70a5eda60ab5744089e2ffe59d71, ASSIGN 2024-11-07T10:47:01,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=db89a463448eb182960960c83f4e0b16, ASSIGN 2024-11-07T10:47:01,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1701fae2bf725fdd94c16d1c6b8bc75f, ASSIGN 2024-11-07T10:47:01,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=257c2c76c61c100547acb4a778fb1fef, ASSIGN 2024-11-07T10:47:01,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=72, state=SUCCESS, hasLock=false; OpenRegionProcedure 257c2c76c61c100547acb4a778fb1fef, server=10641b2c0a69,39013,1730976230620 2024-11-07T10:47:01,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 8f449585c6b7b82f4442604100a49644, server=10641b2c0a69,40573,1730976230476 2024-11-07T10:47:01,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure db89a463448eb182960960c83f4e0b16, server=10641b2c0a69,40573,1730976230476 2024-11-07T10:47:01,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 7a68cc5b6ea7d9e2214ab42d65b16eba, server=10641b2c0a69,40573,1730976230476 2024-11-07T10:47:01,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure 8ecb70a5eda60ab5744089e2ffe59d71, server=10641b2c0a69,40573,1730976230476 2024-11-07T10:47:01,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=SUCCESS, hasLock=false; OpenRegionProcedure 1701fae2bf725fdd94c16d1c6b8bc75f, server=10641b2c0a69,40573,1730976230476 2024-11-07T10:47:01,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=65, state=SUCCESS, hasLock=false; OpenRegionProcedure d4b381fbb1d4cd388f45774fca53b2eb, server=10641b2c0a69,39013,1730976230620 2024-11-07T10:47:01,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure 72bf61b738a2f9b35913a192fc32815b, server=10641b2c0a69,39013,1730976230620 2024-11-07T10:47:01,774 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure 541874b6b96014b10121ede508f1f61f, server=10641b2c0a69,39013,1730976230620 2024-11-07T10:47:01,774 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_FINISH, hasLock=false; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-07T10:47:01,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:01,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 10641b2c0a69%2C41401%2C1730976230578.meta.1730976232360.meta, worker=10641b2c0a69,40573,1730976230476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:01,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:01,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=10641b2c0a69,40573,1730976230476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:01,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 10641b2c0a69%2C41401%2C1730976230578.1730976231927, worker=10641b2c0a69,40573,1730976230476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:01,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 10641b2c0a69%2C41401%2C1730976230578.1730976231927, worker=10641b2c0a69,40573,1730976230476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:01,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8f449585c6b7b82f4442604100a49644, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:01,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d4b381fbb1d4cd388f45774fca53b2eb, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:01,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=72bf61b738a2f9b35913a192fc32815b, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:01,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=67, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=541874b6b96014b10121ede508f1f61f, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:01,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=68, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=7a68cc5b6ea7d9e2214ab42d65b16eba, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:01,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=69, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8ecb70a5eda60ab5744089e2ffe59d71, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:01,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=70, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=db89a463448eb182960960c83f4e0b16, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:01,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=71, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=1701fae2bf725fdd94c16d1c6b8bc75f, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:01,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=257c2c76c61c100547acb4a778fb1fef, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:01,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=72, state=SUCCESS, hasLock=false; OpenRegionProcedure 257c2c76c61c100547acb4a778fb1fef, server=10641b2c0a69,39013,1730976230620 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:01,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 8f449585c6b7b82f4442604100a49644, server=10641b2c0a69,40573,1730976230476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:01,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure db89a463448eb182960960c83f4e0b16, server=10641b2c0a69,40573,1730976230476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:01,775 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 7a68cc5b6ea7d9e2214ab42d65b16eba, server=10641b2c0a69,40573,1730976230476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:01,775 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure 8ecb70a5eda60ab5744089e2ffe59d71, server=10641b2c0a69,40573,1730976230476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:01,775 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=SUCCESS, hasLock=false; OpenRegionProcedure 1701fae2bf725fdd94c16d1c6b8bc75f, server=10641b2c0a69,40573,1730976230476 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:01,775 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=65, state=SUCCESS, hasLock=false; OpenRegionProcedure d4b381fbb1d4cd388f45774fca53b2eb, server=10641b2c0a69,39013,1730976230620 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:01,775 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure 72bf61b738a2f9b35913a192fc32815b, server=10641b2c0a69,39013,1730976230620 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:01,775 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure 541874b6b96014b10121ede508f1f61f, server=10641b2c0a69,39013,1730976230620 didn't hold the lock before restarting, skip acquiring lock. 2024-11-07T10:47:01,775 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-07T10:47:01,776 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=61 2024-11-07T10:47:01,776 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=10641b2c0a69,40573,1730976230476, table=hbase:meta, region=1588230740 2024-11-07T10:47:01,776 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 10641b2c0a69,40573,1730976230476, state=OPEN 2024-11-07T10:47:01,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:47:01,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:47:01,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-07T10:47:01,778 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:47:01,778 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:47:01,778 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-07T10:47:01,782 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1730976451782 2024-11-07T10:47:01,782 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-07T10:47:01,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7a68cc5b6ea7d9e2214ab42d65b16eba, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=5 2024-11-07T10:47:01,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bcc3491e6200809e4c9c7d529da2c4cf, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:47:01,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbe69e720cc4f2a4a4bc41171b6b3c32, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:47:01,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d4b381fbb1d4cd388f45774fca53b2eb, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=5 2024-11-07T10:47:01,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3f6021bbda5b8b7b00318a57f02cd77d, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:47:01,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ddeeab6f2e44d85eb1ee916d3c5e8e9f, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:47:01,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=db89a463448eb182960960c83f4e0b16, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=5 2024-11-07T10:47:01,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c3ab41a5354d389521b694050824788e, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:47:01,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f89575b73864cac8b7b164f417ef378f, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:47:01,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=72bf61b738a2f9b35913a192fc32815b, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=5 2024-11-07T10:47:01,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a7d18696a90cfed5af1fbdb2a706b65, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:47:01,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1baabb362104b46ecb78fe236391ca6f, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:47:01,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ecb70a5eda60ab5744089e2ffe59d71, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=5 2024-11-07T10:47:01,796 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b830813c6b6c3ebcdb59061af10faad, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:47:01,797 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dec39ef6d94f3a519151705fce17adc4, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:47:01,797 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=541874b6b96014b10121ede508f1f61f, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=5 2024-11-07T10:47:01,797 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=64da674e3a9ec8e7458eb07874d9f204, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:47:01,797 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9fba8e9ebafee4e67bfa1295ac44d4a5, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:47:01,797 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1701fae2bf725fdd94c16d1c6b8bc75f, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=5 2024-11-07T10:47:01,797 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cf36f48b84f67b77887428ec6ad5b31, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:47:01,797 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=69160268ea309aa55ece0cd8528b941d, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:47:01,797 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=257c2c76c61c100547acb4a778fb1fef, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=5 2024-11-07T10:47:01,797 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1dd5c615432f6d3976b7d3e591d0a96a, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:47:01,798 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b29f5d988381aa77506fa3b92944854, regionState=OPEN, lastHost=10641b2c0a69,39013,1730976230620, regionLocation=10641b2c0a69,39013,1730976230620, openSeqNum=2 2024-11-07T10:47:01,798 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8f449585c6b7b82f4442604100a49644, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=5 2024-11-07T10:47:01,798 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c71057666e459779c2fa5a4311f4e99c, regionState=OPEN, lastHost=10641b2c0a69,40573,1730976230476, regionLocation=10641b2c0a69,40573,1730976230476, openSeqNum=2 2024-11-07T10:47:01,798 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-07T10:47:01,798 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1730976481798 2024-11-07T10:47:01,798 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1730976541798 2024-11-07T10:47:01,798 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 15 msec 2024-11-07T10:47:01,798 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 10641b2c0a69,41401,1730976230578 after splitting done 2024-11-07T10:47:01,798 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-07T10:47:01,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; ServerCrashProcedure 10641b2c0a69,41401,1730976230578, splitWal=true, meta=true in 3 mins, 4.27 sec 2024-11-07T10:47:02,053 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58538, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-07T10:47:02,057 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 10641b2c0a69:41401 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 10641b2c0a69/172.17.0.2:41401 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-07T10:47:02,058 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba., hostname=10641b2c0a69,41401,1730976230578, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba., hostname=10641b2c0a69,41401,1730976230578, seqNum=2, error=java.net.ConnectException: Call to address=10641b2c0a69:41401 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:47:02,058 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba., hostname=10641b2c0a69,41401,1730976230578, seqNum=2 is java.net.ConnectException: Connection refused 2024-11-07T10:47:02,058 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba., hostname=10641b2c0a69,41401,1730976230578, seqNum=2 from cache 2024-11-07T10:47:02,059 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address 10641b2c0a69:41401 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 10641b2c0a69/172.17.0.2:41401 2024-11-07T10:47:02,168 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 10641b2c0a69:41401 this server is in the failed servers list 2024-11-07T10:47:02,168 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:47:02,169 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:47:02,169 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,41401,1730976230578, seqNum=-1 from cache 2024-11-07T10:47:02,277 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-07T10:47:02,278 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58544, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-07T10:47:02,279 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=10641b2c0a69,40573,1730976230476, seqNum=-1] 2024-11-07T10:47:02,281 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57334, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T10:47:02,285 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba., hostname=10641b2c0a69,40573,1730976230476, seqNum=5] 2024-11-07T10:47:02,312 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38494, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-07T10:47:02,319 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 10641b2c0a69:41401 this server is in the failed servers list 2024-11-07T10:47:02,320 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb., hostname=10641b2c0a69,41401,1730976230578, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb., hostname=10641b2c0a69,41401,1730976230578, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:47:02,320 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb., hostname=10641b2c0a69,41401,1730976230578, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:47:02,320 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb., hostname=10641b2c0a69,41401,1730976230578, seqNum=2 from cache 2024-11-07T10:47:02,432 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb., hostname=10641b2c0a69,39013,1730976230620, seqNum=5] 2024-11-07T10:47:02,457 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.AbstractRpcClient(357): Not trying to connect to 10641b2c0a69:41401 this server is in the failed servers list 2024-11-07T10:47:02,458 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16., hostname=10641b2c0a69,41401,1730976230578, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16., hostname=10641b2c0a69,41401,1730976230578, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:47:02,458 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16., hostname=10641b2c0a69,41401,1730976230578, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:47:02,458 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16., hostname=10641b2c0a69,41401,1730976230578, seqNum=2 from cache 2024-11-07T10:47:02,572 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16., hostname=10641b2c0a69,40573,1730976230476, seqNum=5] 2024-11-07T10:47:02,594 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 10641b2c0a69:41401 this server is in the failed servers list 2024-11-07T10:47:02,595 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b., hostname=10641b2c0a69,41401,1730976230578, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b., hostname=10641b2c0a69,41401,1730976230578, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:47:02,595 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b., hostname=10641b2c0a69,41401,1730976230578, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:47:02,595 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b., hostname=10641b2c0a69,41401,1730976230578, seqNum=2 from cache 2024-11-07T10:47:02,703 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b., hostname=10641b2c0a69,39013,1730976230620, seqNum=5] 2024-11-07T10:47:02,722 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 10641b2c0a69:41401 this server is in the failed servers list 2024-11-07T10:47:02,722 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71., hostname=10641b2c0a69,41401,1730976230578, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71., hostname=10641b2c0a69,41401,1730976230578, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:47:02,722 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71., hostname=10641b2c0a69,41401,1730976230578, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:47:02,722 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71., hostname=10641b2c0a69,41401,1730976230578, seqNum=2 from cache 2024-11-07T10:47:02,833 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71., hostname=10641b2c0a69,40573,1730976230476, seqNum=5] 2024-11-07T10:47:02,844 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 10641b2c0a69:41401 this server is in the failed servers list 2024-11-07T10:47:02,845 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f., hostname=10641b2c0a69,41401,1730976230578, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f., hostname=10641b2c0a69,41401,1730976230578, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:47:02,845 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f., hostname=10641b2c0a69,41401,1730976230578, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:47:02,845 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f., hostname=10641b2c0a69,41401,1730976230578, seqNum=2 from cache 2024-11-07T10:47:02,952 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f., hostname=10641b2c0a69,39013,1730976230620, seqNum=5] 2024-11-07T10:47:02,963 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.AbstractRpcClient(357): Not trying to connect to 10641b2c0a69:41401 this server is in the failed servers list 2024-11-07T10:47:02,964 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f., hostname=10641b2c0a69,41401,1730976230578, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f., hostname=10641b2c0a69,41401,1730976230578, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:47:02,964 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f., hostname=10641b2c0a69,41401,1730976230578, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:47:02,964 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f., hostname=10641b2c0a69,41401,1730976230578, seqNum=2 from cache 2024-11-07T10:47:03,073 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='sss', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f., hostname=10641b2c0a69,40573,1730976230476, seqNum=5] 2024-11-07T10:47:03,088 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 10641b2c0a69:41401 this server is in the failed servers list 2024-11-07T10:47:03,089 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef., hostname=10641b2c0a69,41401,1730976230578, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef., hostname=10641b2c0a69,41401,1730976230578, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:47:03,089 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef., hostname=10641b2c0a69,41401,1730976230578, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:47:03,089 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef., hostname=10641b2c0a69,41401,1730976230578, seqNum=2 from cache 2024-11-07T10:47:03,203 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef., hostname=10641b2c0a69,39013,1730976230620, seqNum=5] 2024-11-07T10:47:03,215 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 10641b2c0a69:41401 this server is in the failed servers list 2024-11-07T10:47:03,215 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644., hostname=10641b2c0a69,41401,1730976230578, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644., hostname=10641b2c0a69,41401,1730976230578, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=10641b2c0a69:41401 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:47:03,215 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644., hostname=10641b2c0a69,41401,1730976230578, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 10641b2c0a69:41401 2024-11-07T10:47:03,215 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644., hostname=10641b2c0a69,41401,1730976230578, seqNum=2 from cache 2024-11-07T10:47:03,324 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yyy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644., hostname=10641b2c0a69,40573,1730976230476, seqNum=5] 2024-11-07T10:47:03,441 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-07T10:47:03,441 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-07T10:47:03,442 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T10:47:03,442 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.procedure.TestSCPBase.tearDown(TestSCPBase.java:72) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T10:47:03,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T10:47:03,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T10:47:03,443 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-07T10:47:03,443 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-07T10:47:03,443 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=431525349, stopped=false 2024-11-07T10:47:03,443 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=10641b2c0a69,43605,1730976229810 2024-11-07T10:47:03,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T10:47:03,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:47:03,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T10:47:03,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:47:03,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-07T10:47:03,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:47:03,446 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T10:47:03,447 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-07T10:47:03,447 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T10:47:03,447 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T10:47:03,447 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(878): Closing user regions 2024-11-07T10:47:03,448 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(3091): Received CLOSE for 2cf36f48b84f67b77887428ec6ad5b31 2024-11-07T10:47:03,448 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.procedure.TestSCPBase.tearDown(TestSCPBase.java:72) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T10:47:03,448 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T10:47:03,448 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-07T10:47:03,449 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '10641b2c0a69,40573,1730976230476' ***** 2024-11-07T10:47:03,449 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-07T10:47:03,449 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '10641b2c0a69,39013,1730976230620' ***** 2024-11-07T10:47:03,449 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-07T10:47:03,449 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-07T10:47:03,449 INFO [RS:2;10641b2c0a69:39013 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-07T10:47:03,449 INFO [RS:2;10641b2c0a69:39013 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-07T10:47:03,449 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-07T10:47:03,449 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(3091): Received CLOSE for 257c2c76c61c100547acb4a778fb1fef 2024-11-07T10:47:03,452 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(3091): Received CLOSE for 9fba8e9ebafee4e67bfa1295ac44d4a5 2024-11-07T10:47:03,453 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(3091): Received CLOSE for 1701fae2bf725fdd94c16d1c6b8bc75f 2024-11-07T10:47:03,453 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(3091): Received CLOSE for bcc3491e6200809e4c9c7d529da2c4cf 2024-11-07T10:47:03,453 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(3091): Received CLOSE for 8ecb70a5eda60ab5744089e2ffe59d71 2024-11-07T10:47:03,453 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(3091): Received CLOSE for 1dd5c615432f6d3976b7d3e591d0a96a 2024-11-07T10:47:03,453 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(3091): Received CLOSE for 8f449585c6b7b82f4442604100a49644 2024-11-07T10:47:03,453 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(3091): Received CLOSE for db89a463448eb182960960c83f4e0b16 2024-11-07T10:47:03,453 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(3091): Received CLOSE for 7a68cc5b6ea7d9e2214ab42d65b16eba 2024-11-07T10:47:03,453 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(3091): Received CLOSE for ddeeab6f2e44d85eb1ee916d3c5e8e9f 2024-11-07T10:47:03,453 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(3091): Received CLOSE for 9b830813c6b6c3ebcdb59061af10faad 2024-11-07T10:47:03,453 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(3091): Received CLOSE for c71057666e459779c2fa5a4311f4e99c 2024-11-07T10:47:03,453 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(3091): Received CLOSE for 0a7d18696a90cfed5af1fbdb2a706b65 2024-11-07T10:47:03,453 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(3091): Received CLOSE for c3ab41a5354d389521b694050824788e 2024-11-07T10:47:03,453 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2cf36f48b84f67b77887428ec6ad5b31, disabling compactions & flushes 2024-11-07T10:47:03,454 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1730976232769.2cf36f48b84f67b77887428ec6ad5b31. 2024-11-07T10:47:03,454 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1730976232769.2cf36f48b84f67b77887428ec6ad5b31. 2024-11-07T10:47:03,454 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1730976232769.2cf36f48b84f67b77887428ec6ad5b31. after waiting 0 ms 2024-11-07T10:47:03,454 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1730976232769.2cf36f48b84f67b77887428ec6ad5b31. 2024-11-07T10:47:03,454 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 2cf36f48b84f67b77887428ec6ad5b31 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-07T10:47:03,458 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-07T10:47:03,458 INFO [RS:0;10641b2c0a69:40573 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-07T10:47:03,458 INFO [RS:0;10641b2c0a69:40573 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-07T10:47:03,458 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-07T10:47:03,458 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(3093): Received CLOSE for the region: 9fba8e9ebafee4e67bfa1295ac44d4a5, which we are already trying to CLOSE, but not completed yet 2024-11-07T10:47:03,458 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(3093): Received CLOSE for the region: 1701fae2bf725fdd94c16d1c6b8bc75f, which we are already trying to CLOSE, but not completed yet 2024-11-07T10:47:03,458 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(3093): Received CLOSE for the region: bcc3491e6200809e4c9c7d529da2c4cf, which we are already trying to CLOSE, but not completed yet 2024-11-07T10:47:03,458 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(3093): Received CLOSE for the region: 8ecb70a5eda60ab5744089e2ffe59d71, which we are already trying to CLOSE, but not completed yet 2024-11-07T10:47:03,458 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(3093): Received CLOSE for the region: 1dd5c615432f6d3976b7d3e591d0a96a, which we are already trying to CLOSE, but not completed yet 2024-11-07T10:47:03,458 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(3093): Received CLOSE for the region: 8f449585c6b7b82f4442604100a49644, which we are already trying to CLOSE, but not completed yet 2024-11-07T10:47:03,458 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(3093): Received CLOSE for the region: db89a463448eb182960960c83f4e0b16, which we are already trying to CLOSE, but not completed yet 2024-11-07T10:47:03,458 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(3093): Received CLOSE for the region: 7a68cc5b6ea7d9e2214ab42d65b16eba, which we are already trying to CLOSE, but not completed yet 2024-11-07T10:47:03,458 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(3093): Received CLOSE for the region: ddeeab6f2e44d85eb1ee916d3c5e8e9f, which we are already trying to CLOSE, but not completed yet 2024-11-07T10:47:03,458 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(3093): Received CLOSE for the region: 9b830813c6b6c3ebcdb59061af10faad, which we are already trying to CLOSE, but not completed yet 2024-11-07T10:47:03,458 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(3093): Received CLOSE for the region: c71057666e459779c2fa5a4311f4e99c, which we are already trying to CLOSE, but not completed yet 2024-11-07T10:47:03,458 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(3093): Received CLOSE for the region: 0a7d18696a90cfed5af1fbdb2a706b65, which we are already trying to CLOSE, but not completed yet 2024-11-07T10:47:03,458 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(3093): Received CLOSE for the region: c3ab41a5354d389521b694050824788e, which we are already trying to CLOSE, but not completed yet 2024-11-07T10:47:03,459 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(959): stopping server 10641b2c0a69,40573,1730976230476 2024-11-07T10:47:03,459 INFO [RS:0;10641b2c0a69:40573 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T10:47:03,459 INFO [RS:0;10641b2c0a69:40573 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;10641b2c0a69:40573. 2024-11-07T10:47:03,459 DEBUG [RS:0;10641b2c0a69:40573 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T10:47:03,459 DEBUG [RS:0;10641b2c0a69:40573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T10:47:03,459 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-07T10:47:03,459 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-07T10:47:03,459 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-07T10:47:03,459 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-07T10:47:03,460 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(3091): Received CLOSE for 64da674e3a9ec8e7458eb07874d9f204 2024-11-07T10:47:03,460 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(3091): Received CLOSE for 69160268ea309aa55ece0cd8528b941d 2024-11-07T10:47:03,461 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(3091): Received CLOSE for 1baabb362104b46ecb78fe236391ca6f 2024-11-07T10:47:03,461 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(3091): Received CLOSE for d4b381fbb1d4cd388f45774fca53b2eb 2024-11-07T10:47:03,461 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(3091): Received CLOSE for dbe69e720cc4f2a4a4bc41171b6b3c32 2024-11-07T10:47:03,461 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(3091): Received CLOSE for dec39ef6d94f3a519151705fce17adc4 2024-11-07T10:47:03,461 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(3091): Received CLOSE for 72bf61b738a2f9b35913a192fc32815b 2024-11-07T10:47:03,461 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 257c2c76c61c100547acb4a778fb1fef, disabling compactions & flushes 2024-11-07T10:47:03,461 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(3091): Received CLOSE for f89575b73864cac8b7b164f417ef378f 2024-11-07T10:47:03,461 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(3091): Received CLOSE for 0b29f5d988381aa77506fa3b92944854 2024-11-07T10:47:03,461 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef. 2024-11-07T10:47:03,461 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(3091): Received CLOSE for 541874b6b96014b10121ede508f1f61f 2024-11-07T10:47:03,461 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef. 2024-11-07T10:47:03,461 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(3091): Received CLOSE for 3f6021bbda5b8b7b00318a57f02cd77d 2024-11-07T10:47:03,461 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(959): stopping server 10641b2c0a69,39013,1730976230620 2024-11-07T10:47:03,461 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef. after waiting 0 ms 2024-11-07T10:47:03,461 INFO [RS:2;10641b2c0a69:39013 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T10:47:03,461 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef. 2024-11-07T10:47:03,461 INFO [RS:2;10641b2c0a69:39013 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;10641b2c0a69:39013. 2024-11-07T10:47:03,461 DEBUG [RS:2;10641b2c0a69:39013 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-07T10:47:03,461 DEBUG [RS:2;10641b2c0a69:39013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T10:47:03,461 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(1321): Waiting on 12 regions to close 2024-11-07T10:47:03,462 DEBUG [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(1351): Waiting on 0b29f5d988381aa77506fa3b92944854, 1baabb362104b46ecb78fe236391ca6f, 257c2c76c61c100547acb4a778fb1fef, 3f6021bbda5b8b7b00318a57f02cd77d, 541874b6b96014b10121ede508f1f61f, 64da674e3a9ec8e7458eb07874d9f204, 69160268ea309aa55ece0cd8528b941d, 72bf61b738a2f9b35913a192fc32815b, d4b381fbb1d4cd388f45774fca53b2eb, dbe69e720cc4f2a4a4bc41171b6b3c32, dec39ef6d94f3a519151705fce17adc4, f89575b73864cac8b7b164f417ef378f 2024-11-07T10:47:03,464 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(1321): Waiting on 15 regions to close 2024-11-07T10:47:03,464 DEBUG [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(1351): Waiting on 0a7d18696a90cfed5af1fbdb2a706b65, 1588230740, 1701fae2bf725fdd94c16d1c6b8bc75f, 1dd5c615432f6d3976b7d3e591d0a96a, 2cf36f48b84f67b77887428ec6ad5b31, 7a68cc5b6ea7d9e2214ab42d65b16eba, 8ecb70a5eda60ab5744089e2ffe59d71, 8f449585c6b7b82f4442604100a49644, 9b830813c6b6c3ebcdb59061af10faad, 9fba8e9ebafee4e67bfa1295ac44d4a5, bcc3491e6200809e4c9c7d529da2c4cf, c3ab41a5354d389521b694050824788e, c71057666e459779c2fa5a4311f4e99c, db89a463448eb182960960c83f4e0b16, ddeeab6f2e44d85eb1ee916d3c5e8e9f 2024-11-07T10:47:03,465 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-07T10:47:03,465 INFO [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-07T10:47:03,465 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-07T10:47:03,465 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-07T10:47:03,465 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-07T10:47:03,465 INFO [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=13.37 KB heapSize=19.67 KB 2024-11-07T10:47:03,480 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/.tmp/info/f7dddeb4334f4946a8432ace9535caba is 268, key is testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb./info:regioninfo/1730976420959/Put/seqid=0 2024-11-07T10:47:03,496 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/257c2c76c61c100547acb4a778fb1fef/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-07T10:47:03,497 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cf36f48b84f67b77887428ec6ad5b31/.tmp/colfamily11/cd6d9818c0de434d9912310362dc71fc is 52, key is ttt/colfamily11:colfamily11/1730976236581/Put/seqid=0 2024-11-07T10:47:03,501 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef. 2024-11-07T10:47:03,501 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 257c2c76c61c100547acb4a778fb1fef: Waiting for close lock at 1730976423461Running coprocessor pre-close hooks at 1730976423461Disabling compacts and flushes for region at 1730976423461Disabling writes for close at 1730976423461Writing region close event to WAL at 1730976423477 (+16 ms)Running coprocessor post-close hooks at 1730976423497 (+20 ms)Closed at 1730976423501 (+4 ms) 2024-11-07T10:47:03,502 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1730976232769.257c2c76c61c100547acb4a778fb1fef. 2024-11-07T10:47:03,502 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 64da674e3a9ec8e7458eb07874d9f204, disabling compactions & flushes 2024-11-07T10:47:03,502 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1730976232769.64da674e3a9ec8e7458eb07874d9f204. 2024-11-07T10:47:03,502 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1730976232769.64da674e3a9ec8e7458eb07874d9f204. 2024-11-07T10:47:03,502 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1730976232769.64da674e3a9ec8e7458eb07874d9f204. after waiting 0 ms 2024-11-07T10:47:03,502 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1730976232769.64da674e3a9ec8e7458eb07874d9f204. 2024-11-07T10:47:03,502 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 64da674e3a9ec8e7458eb07874d9f204 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-07T10:47:03,525 INFO [regionserver/10641b2c0a69:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T10:47:03,525 INFO [regionserver/10641b2c0a69:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T10:47:03,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741888_1064 (size=39263) 2024-11-07T10:47:03,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741888_1064 (size=39263) 2024-11-07T10:47:03,531 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64da674e3a9ec8e7458eb07874d9f204/.tmp/colfamily11/df74bf9da00f4ff59c032fce7383ffff is 52, key is qqq/colfamily11:colfamily11/1730976236776/Put/seqid=0 2024-11-07T10:47:03,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741888_1064 (size=39263) 2024-11-07T10:47:03,533 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cf36f48b84f67b77887428ec6ad5b31/.tmp/colfamily11/cd6d9818c0de434d9912310362dc71fc 2024-11-07T10:47:03,542 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cf36f48b84f67b77887428ec6ad5b31/.tmp/colfamily11/cd6d9818c0de434d9912310362dc71fc as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cf36f48b84f67b77887428ec6ad5b31/colfamily11/cd6d9818c0de434d9912310362dc71fc 2024-11-07T10:47:03,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741887_1063 (size=19812) 2024-11-07T10:47:03,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741887_1063 (size=19812) 2024-11-07T10:47:03,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741887_1063 (size=19812) 2024-11-07T10:47:03,546 INFO [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.37 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/.tmp/info/f7dddeb4334f4946a8432ace9535caba 2024-11-07T10:47:03,551 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cf36f48b84f67b77887428ec6ad5b31/colfamily11/cd6d9818c0de434d9912310362dc71fc, entries=703, sequenceid=5, filesize=38.3 K 2024-11-07T10:47:03,553 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 2cf36f48b84f67b77887428ec6ad5b31 in 98ms, sequenceid=5, compaction requested=false 2024-11-07T10:47:03,554 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/.tmp/info/f7dddeb4334f4946a8432ace9535caba as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/info/f7dddeb4334f4946a8432ace9535caba 2024-11-07T10:47:03,564 INFO [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/info/f7dddeb4334f4946a8432ace9535caba, entries=72, sequenceid=81, filesize=19.3 K 2024-11-07T10:47:03,566 INFO [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~13.37 KB/13686, heapSize ~18.91 KB/19360, currentSize=0 B/0 for 1588230740 in 100ms, sequenceid=81, compaction requested=false 2024-11-07T10:47:03,587 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cf36f48b84f67b77887428ec6ad5b31/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-07T10:47:03,588 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1730976232769.2cf36f48b84f67b77887428ec6ad5b31. 2024-11-07T10:47:03,588 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2cf36f48b84f67b77887428ec6ad5b31: Waiting for close lock at 1730976423453Running coprocessor pre-close hooks at 1730976423453Disabling compacts and flushes for region at 1730976423453Disabling writes for close at 1730976423454 (+1 ms)Obtaining lock to block concurrent updates at 1730976423454Preparing flush snapshotting stores in 2cf36f48b84f67b77887428ec6ad5b31 at 1730976423454Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1730976232769.2cf36f48b84f67b77887428ec6ad5b31., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1730976423462 (+8 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1730976232769.2cf36f48b84f67b77887428ec6ad5b31. at 1730976423464 (+2 ms)Flushing 2cf36f48b84f67b77887428ec6ad5b31/colfamily11: creating writer at 1730976423464Flushing 2cf36f48b84f67b77887428ec6ad5b31/colfamily11: appending metadata at 1730976423497 (+33 ms)Flushing 2cf36f48b84f67b77887428ec6ad5b31/colfamily11: closing flushed file at 1730976423497Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@707e554b: reopening flushed file at 1730976423541 (+44 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 2cf36f48b84f67b77887428ec6ad5b31 in 98ms, sequenceid=5, compaction requested=false at 1730976423553 (+12 ms)Writing region close event to WAL at 1730976423578 (+25 ms)Running coprocessor post-close hooks at 1730976423588 (+10 ms)Closed at 1730976423588 2024-11-07T10:47:03,588 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1730976232769.2cf36f48b84f67b77887428ec6ad5b31. 2024-11-07T10:47:03,588 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9fba8e9ebafee4e67bfa1295ac44d4a5, disabling compactions & flushes 2024-11-07T10:47:03,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741889_1065 (size=39263) 2024-11-07T10:47:03,589 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1730976232769.9fba8e9ebafee4e67bfa1295ac44d4a5. 2024-11-07T10:47:03,589 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1730976232769.9fba8e9ebafee4e67bfa1295ac44d4a5. 2024-11-07T10:47:03,589 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1730976232769.9fba8e9ebafee4e67bfa1295ac44d4a5. after waiting 0 ms 2024-11-07T10:47:03,589 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1730976232769.9fba8e9ebafee4e67bfa1295ac44d4a5. 2024-11-07T10:47:03,589 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 9fba8e9ebafee4e67bfa1295ac44d4a5 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-07T10:47:03,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741889_1065 (size=39263) 2024-11-07T10:47:03,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741889_1065 (size=39263) 2024-11-07T10:47:03,590 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64da674e3a9ec8e7458eb07874d9f204/.tmp/colfamily11/df74bf9da00f4ff59c032fce7383ffff 2024-11-07T10:47:03,593 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-07T10:47:03,599 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64da674e3a9ec8e7458eb07874d9f204/.tmp/colfamily11/df74bf9da00f4ff59c032fce7383ffff as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64da674e3a9ec8e7458eb07874d9f204/colfamily11/df74bf9da00f4ff59c032fce7383ffff 2024-11-07T10:47:03,610 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/hbase/meta/1588230740/recovered.edits/84.seqid, newMaxSeqId=84, maxSeqId=60 2024-11-07T10:47:03,611 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-07T10:47:03,611 INFO [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-07T10:47:03,612 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1730976423465Running coprocessor pre-close hooks at 1730976423465Disabling compacts and flushes for region at 1730976423465Disabling writes for close at 1730976423465Obtaining lock to block concurrent updates at 1730976423465Preparing flush snapshotting stores in 1588230740 at 1730976423465Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=13686, getHeapSize=20080, getOffHeapSize=0, getCellsCount=72 at 1730976423465Flushing stores of hbase:meta,,1.1588230740 at 1730976423466 (+1 ms)Flushing 1588230740/info: creating writer at 1730976423466Flushing 1588230740/info: appending metadata at 1730976423479 (+13 ms)Flushing 1588230740/info: closing flushed file at 1730976423479Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46c41176: reopening flushed file at 1730976423553 (+74 ms)Finished flush of dataSize ~13.37 KB/13686, heapSize ~18.91 KB/19360, currentSize=0 B/0 for 1588230740 in 100ms, sequenceid=81, compaction requested=false at 1730976423566 (+13 ms)Writing region close event to WAL at 1730976423587 (+21 ms)Running coprocessor post-close hooks at 1730976423611 (+24 ms)Closed at 1730976423611 2024-11-07T10:47:03,612 DEBUG [RS_CLOSE_META-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-07T10:47:03,612 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64da674e3a9ec8e7458eb07874d9f204/colfamily11/df74bf9da00f4ff59c032fce7383ffff, entries=703, sequenceid=5, filesize=38.3 K 2024-11-07T10:47:03,613 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 64da674e3a9ec8e7458eb07874d9f204 in 111ms, sequenceid=5, compaction requested=false 2024-11-07T10:47:03,616 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9fba8e9ebafee4e67bfa1295ac44d4a5/.tmp/colfamily11/c84a1fd97d1a4ca3a229fb340cdb206b is 52, key is rrr/colfamily11:colfamily11/1730976236538/Put/seqid=0 2024-11-07T10:47:03,629 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/64da674e3a9ec8e7458eb07874d9f204/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-07T10:47:03,629 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1730976232769.64da674e3a9ec8e7458eb07874d9f204. 2024-11-07T10:47:03,630 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 64da674e3a9ec8e7458eb07874d9f204: Waiting for close lock at 1730976423502Running coprocessor pre-close hooks at 1730976423502Disabling compacts and flushes for region at 1730976423502Disabling writes for close at 1730976423502Obtaining lock to block concurrent updates at 1730976423502Preparing flush snapshotting stores in 64da674e3a9ec8e7458eb07874d9f204 at 1730976423502Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1730976232769.64da674e3a9ec8e7458eb07874d9f204., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1730976423502Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1730976232769.64da674e3a9ec8e7458eb07874d9f204. at 1730976423503 (+1 ms)Flushing 64da674e3a9ec8e7458eb07874d9f204/colfamily11: creating writer at 1730976423503Flushing 64da674e3a9ec8e7458eb07874d9f204/colfamily11: appending metadata at 1730976423530 (+27 ms)Flushing 64da674e3a9ec8e7458eb07874d9f204/colfamily11: closing flushed file at 1730976423530Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d34f43: reopening flushed file at 1730976423598 (+68 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 64da674e3a9ec8e7458eb07874d9f204 in 111ms, sequenceid=5, compaction requested=false at 1730976423613 (+15 ms)Writing region close event to WAL at 1730976423621 (+8 ms)Running coprocessor post-close hooks at 1730976423629 (+8 ms)Closed at 1730976423629 2024-11-07T10:47:03,630 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1730976232769.64da674e3a9ec8e7458eb07874d9f204. 2024-11-07T10:47:03,630 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 69160268ea309aa55ece0cd8528b941d, disabling compactions & flushes 2024-11-07T10:47:03,630 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1730976232769.69160268ea309aa55ece0cd8528b941d. 2024-11-07T10:47:03,630 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1730976232769.69160268ea309aa55ece0cd8528b941d. 2024-11-07T10:47:03,630 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1730976232769.69160268ea309aa55ece0cd8528b941d. after waiting 0 ms 2024-11-07T10:47:03,630 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1730976232769.69160268ea309aa55ece0cd8528b941d. 2024-11-07T10:47:03,630 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 69160268ea309aa55ece0cd8528b941d 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-07T10:47:03,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741890_1066 (size=39263) 2024-11-07T10:47:03,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741890_1066 (size=39263) 2024-11-07T10:47:03,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741890_1066 (size=39263) 2024-11-07T10:47:03,661 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/69160268ea309aa55ece0cd8528b941d/.tmp/colfamily11/eb88b1c38ad540dd80d683eb8923406d is 52, key is uuu/colfamily11:colfamily11/1730976236816/Put/seqid=0 2024-11-07T10:47:03,661 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9fba8e9ebafee4e67bfa1295ac44d4a5/.tmp/colfamily11/c84a1fd97d1a4ca3a229fb340cdb206b 2024-11-07T10:47:03,662 DEBUG [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(1351): Waiting on 0b29f5d988381aa77506fa3b92944854, 1baabb362104b46ecb78fe236391ca6f, 3f6021bbda5b8b7b00318a57f02cd77d, 541874b6b96014b10121ede508f1f61f, 69160268ea309aa55ece0cd8528b941d, 72bf61b738a2f9b35913a192fc32815b, d4b381fbb1d4cd388f45774fca53b2eb, dbe69e720cc4f2a4a4bc41171b6b3c32, dec39ef6d94f3a519151705fce17adc4, f89575b73864cac8b7b164f417ef378f 2024-11-07T10:47:03,665 DEBUG [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(1351): Waiting on 0a7d18696a90cfed5af1fbdb2a706b65, 1701fae2bf725fdd94c16d1c6b8bc75f, 1dd5c615432f6d3976b7d3e591d0a96a, 7a68cc5b6ea7d9e2214ab42d65b16eba, 8ecb70a5eda60ab5744089e2ffe59d71, 8f449585c6b7b82f4442604100a49644, 9b830813c6b6c3ebcdb59061af10faad, 9fba8e9ebafee4e67bfa1295ac44d4a5, bcc3491e6200809e4c9c7d529da2c4cf, c3ab41a5354d389521b694050824788e, c71057666e459779c2fa5a4311f4e99c, db89a463448eb182960960c83f4e0b16, ddeeab6f2e44d85eb1ee916d3c5e8e9f 2024-11-07T10:47:03,668 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9fba8e9ebafee4e67bfa1295ac44d4a5/.tmp/colfamily11/c84a1fd97d1a4ca3a229fb340cdb206b as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9fba8e9ebafee4e67bfa1295ac44d4a5/colfamily11/c84a1fd97d1a4ca3a229fb340cdb206b 2024-11-07T10:47:03,674 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9fba8e9ebafee4e67bfa1295ac44d4a5/colfamily11/c84a1fd97d1a4ca3a229fb340cdb206b, entries=703, sequenceid=5, filesize=38.3 K 2024-11-07T10:47:03,676 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 9fba8e9ebafee4e67bfa1295ac44d4a5 in 86ms, sequenceid=5, compaction requested=false 2024-11-07T10:47:03,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741891_1067 (size=39263) 2024-11-07T10:47:03,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741891_1067 (size=39263) 2024-11-07T10:47:03,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741891_1067 (size=39263) 2024-11-07T10:47:03,685 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/69160268ea309aa55ece0cd8528b941d/.tmp/colfamily11/eb88b1c38ad540dd80d683eb8923406d 2024-11-07T10:47:03,687 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9fba8e9ebafee4e67bfa1295ac44d4a5/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-07T10:47:03,688 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1730976232769.9fba8e9ebafee4e67bfa1295ac44d4a5. 2024-11-07T10:47:03,688 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9fba8e9ebafee4e67bfa1295ac44d4a5: Waiting for close lock at 1730976423588Running coprocessor pre-close hooks at 1730976423588Disabling compacts and flushes for region at 1730976423588Disabling writes for close at 1730976423589 (+1 ms)Obtaining lock to block concurrent updates at 1730976423589Preparing flush snapshotting stores in 9fba8e9ebafee4e67bfa1295ac44d4a5 at 1730976423589Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1730976232769.9fba8e9ebafee4e67bfa1295ac44d4a5., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1730976423589Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1730976232769.9fba8e9ebafee4e67bfa1295ac44d4a5. at 1730976423593 (+4 ms)Flushing 9fba8e9ebafee4e67bfa1295ac44d4a5/colfamily11: creating writer at 1730976423593Flushing 9fba8e9ebafee4e67bfa1295ac44d4a5/colfamily11: appending metadata at 1730976423615 (+22 ms)Flushing 9fba8e9ebafee4e67bfa1295ac44d4a5/colfamily11: closing flushed file at 1730976423616 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3293b457: reopening flushed file at 1730976423668 (+52 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 9fba8e9ebafee4e67bfa1295ac44d4a5 in 86ms, sequenceid=5, compaction requested=false at 1730976423676 (+8 ms)Writing region close event to WAL at 1730976423682 (+6 ms)Running coprocessor post-close hooks at 1730976423688 (+6 ms)Closed at 1730976423688 2024-11-07T10:47:03,689 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1730976232769.9fba8e9ebafee4e67bfa1295ac44d4a5. 2024-11-07T10:47:03,689 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1701fae2bf725fdd94c16d1c6b8bc75f, disabling compactions & flushes 2024-11-07T10:47:03,689 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f. 2024-11-07T10:47:03,689 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f. 2024-11-07T10:47:03,689 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f. after waiting 0 ms 2024-11-07T10:47:03,689 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f. 2024-11-07T10:47:03,693 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/69160268ea309aa55ece0cd8528b941d/.tmp/colfamily11/eb88b1c38ad540dd80d683eb8923406d as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/69160268ea309aa55ece0cd8528b941d/colfamily11/eb88b1c38ad540dd80d683eb8923406d 2024-11-07T10:47:03,695 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1701fae2bf725fdd94c16d1c6b8bc75f/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-07T10:47:03,696 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f. 2024-11-07T10:47:03,696 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1701fae2bf725fdd94c16d1c6b8bc75f: Waiting for close lock at 1730976423689Running coprocessor pre-close hooks at 1730976423689Disabling compacts and flushes for region at 1730976423689Disabling writes for close at 1730976423689Writing region close event to WAL at 1730976423690 (+1 ms)Running coprocessor post-close hooks at 1730976423696 (+6 ms)Closed at 1730976423696 2024-11-07T10:47:03,696 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1730976232769.1701fae2bf725fdd94c16d1c6b8bc75f. 2024-11-07T10:47:03,696 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bcc3491e6200809e4c9c7d529da2c4cf, disabling compactions & flushes 2024-11-07T10:47:03,697 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1730976232769.bcc3491e6200809e4c9c7d529da2c4cf. 2024-11-07T10:47:03,697 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1730976232769.bcc3491e6200809e4c9c7d529da2c4cf. 2024-11-07T10:47:03,697 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1730976232769.bcc3491e6200809e4c9c7d529da2c4cf. after waiting 0 ms 2024-11-07T10:47:03,697 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1730976232769.bcc3491e6200809e4c9c7d529da2c4cf. 2024-11-07T10:47:03,697 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing bcc3491e6200809e4c9c7d529da2c4cf 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-07T10:47:03,702 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/69160268ea309aa55ece0cd8528b941d/colfamily11/eb88b1c38ad540dd80d683eb8923406d, entries=703, sequenceid=5, filesize=38.3 K 2024-11-07T10:47:03,703 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 69160268ea309aa55ece0cd8528b941d in 73ms, sequenceid=5, compaction requested=false 2024-11-07T10:47:03,721 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/69160268ea309aa55ece0cd8528b941d/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-07T10:47:03,722 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1730976232769.69160268ea309aa55ece0cd8528b941d. 2024-11-07T10:47:03,722 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 69160268ea309aa55ece0cd8528b941d: Waiting for close lock at 1730976423630Running coprocessor pre-close hooks at 1730976423630Disabling compacts and flushes for region at 1730976423630Disabling writes for close at 1730976423630Obtaining lock to block concurrent updates at 1730976423630Preparing flush snapshotting stores in 69160268ea309aa55ece0cd8528b941d at 1730976423630Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1730976232769.69160268ea309aa55ece0cd8528b941d., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1730976423631 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1730976232769.69160268ea309aa55ece0cd8528b941d. at 1730976423631Flushing 69160268ea309aa55ece0cd8528b941d/colfamily11: creating writer at 1730976423631Flushing 69160268ea309aa55ece0cd8528b941d/colfamily11: appending metadata at 1730976423661 (+30 ms)Flushing 69160268ea309aa55ece0cd8528b941d/colfamily11: closing flushed file at 1730976423661Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a279fcc: reopening flushed file at 1730976423692 (+31 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 69160268ea309aa55ece0cd8528b941d in 73ms, sequenceid=5, compaction requested=false at 1730976423703 (+11 ms)Writing region close event to WAL at 1730976423714 (+11 ms)Running coprocessor post-close hooks at 1730976423722 (+8 ms)Closed at 1730976423722 2024-11-07T10:47:03,722 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1730976232769.69160268ea309aa55ece0cd8528b941d. 2024-11-07T10:47:03,722 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1baabb362104b46ecb78fe236391ca6f, disabling compactions & flushes 2024-11-07T10:47:03,722 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1730976232769.1baabb362104b46ecb78fe236391ca6f. 2024-11-07T10:47:03,722 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1730976232769.1baabb362104b46ecb78fe236391ca6f. 2024-11-07T10:47:03,722 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1730976232769.1baabb362104b46ecb78fe236391ca6f. after waiting 0 ms 2024-11-07T10:47:03,722 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1730976232769.1baabb362104b46ecb78fe236391ca6f. 2024-11-07T10:47:03,722 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1baabb362104b46ecb78fe236391ca6f 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-07T10:47:03,723 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcc3491e6200809e4c9c7d529da2c4cf/.tmp/colfamily11/fdedfd30a35446acbc2a3c82675c10ae is 52, key is bbb/colfamily11:colfamily11/1730976236269/Put/seqid=0 2024-11-07T10:47:03,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741892_1068 (size=39263) 2024-11-07T10:47:03,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741892_1068 (size=39263) 2024-11-07T10:47:03,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741892_1068 (size=39263) 2024-11-07T10:47:03,749 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcc3491e6200809e4c9c7d529da2c4cf/.tmp/colfamily11/fdedfd30a35446acbc2a3c82675c10ae 2024-11-07T10:47:03,756 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1baabb362104b46ecb78fe236391ca6f/.tmp/colfamily11/44c57c7514b84fd6a5d3fa810f4b7149 is 52, key is lll/colfamily11:colfamily11/1730976236698/Put/seqid=0 2024-11-07T10:47:03,757 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcc3491e6200809e4c9c7d529da2c4cf/.tmp/colfamily11/fdedfd30a35446acbc2a3c82675c10ae as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcc3491e6200809e4c9c7d529da2c4cf/colfamily11/fdedfd30a35446acbc2a3c82675c10ae 2024-11-07T10:47:03,760 INFO [regionserver/10641b2c0a69:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-07T10:47:03,760 INFO [regionserver/10641b2c0a69:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-07T10:47:03,762 INFO [regionserver/10641b2c0a69:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-07T10:47:03,762 INFO [regionserver/10641b2c0a69:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-07T10:47:03,765 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcc3491e6200809e4c9c7d529da2c4cf/colfamily11/fdedfd30a35446acbc2a3c82675c10ae, entries=703, sequenceid=5, filesize=38.3 K 2024-11-07T10:47:03,766 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for bcc3491e6200809e4c9c7d529da2c4cf in 69ms, sequenceid=5, compaction requested=false 2024-11-07T10:47:03,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741893_1069 (size=39263) 2024-11-07T10:47:03,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741893_1069 (size=39263) 2024-11-07T10:47:03,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741893_1069 (size=39263) 2024-11-07T10:47:03,793 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bcc3491e6200809e4c9c7d529da2c4cf/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-07T10:47:03,794 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1730976232769.bcc3491e6200809e4c9c7d529da2c4cf. 2024-11-07T10:47:03,794 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bcc3491e6200809e4c9c7d529da2c4cf: Waiting for close lock at 1730976423696Running coprocessor pre-close hooks at 1730976423696Disabling compacts and flushes for region at 1730976423696Disabling writes for close at 1730976423697 (+1 ms)Obtaining lock to block concurrent updates at 1730976423697Preparing flush snapshotting stores in bcc3491e6200809e4c9c7d529da2c4cf at 1730976423697Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1730976232769.bcc3491e6200809e4c9c7d529da2c4cf., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1730976423697Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1730976232769.bcc3491e6200809e4c9c7d529da2c4cf. at 1730976423698 (+1 ms)Flushing bcc3491e6200809e4c9c7d529da2c4cf/colfamily11: creating writer at 1730976423699 (+1 ms)Flushing bcc3491e6200809e4c9c7d529da2c4cf/colfamily11: appending metadata at 1730976423723 (+24 ms)Flushing bcc3491e6200809e4c9c7d529da2c4cf/colfamily11: closing flushed file at 1730976423723Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31254787: reopening flushed file at 1730976423756 (+33 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for bcc3491e6200809e4c9c7d529da2c4cf in 69ms, sequenceid=5, compaction requested=false at 1730976423766 (+10 ms)Writing region close event to WAL at 1730976423778 (+12 ms)Running coprocessor post-close hooks at 1730976423794 (+16 ms)Closed at 1730976423794 2024-11-07T10:47:03,794 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1730976232769.bcc3491e6200809e4c9c7d529da2c4cf. 2024-11-07T10:47:03,795 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8ecb70a5eda60ab5744089e2ffe59d71, disabling compactions & flushes 2024-11-07T10:47:03,795 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71. 2024-11-07T10:47:03,795 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71. 2024-11-07T10:47:03,795 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71. after waiting 0 ms 2024-11-07T10:47:03,795 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71. 2024-11-07T10:47:03,805 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8ecb70a5eda60ab5744089e2ffe59d71/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-07T10:47:03,805 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71. 2024-11-07T10:47:03,805 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8ecb70a5eda60ab5744089e2ffe59d71: Waiting for close lock at 1730976423795Running coprocessor pre-close hooks at 1730976423795Disabling compacts and flushes for region at 1730976423795Disabling writes for close at 1730976423795Writing region close event to WAL at 1730976423796 (+1 ms)Running coprocessor post-close hooks at 1730976423805 (+9 ms)Closed at 1730976423805 2024-11-07T10:47:03,806 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1730976232769.8ecb70a5eda60ab5744089e2ffe59d71. 2024-11-07T10:47:03,806 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1dd5c615432f6d3976b7d3e591d0a96a, disabling compactions & flushes 2024-11-07T10:47:03,806 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1730976232769.1dd5c615432f6d3976b7d3e591d0a96a. 2024-11-07T10:47:03,806 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1730976232769.1dd5c615432f6d3976b7d3e591d0a96a. 2024-11-07T10:47:03,806 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1730976232769.1dd5c615432f6d3976b7d3e591d0a96a. after waiting 0 ms 2024-11-07T10:47:03,806 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1730976232769.1dd5c615432f6d3976b7d3e591d0a96a. 2024-11-07T10:47:03,806 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1dd5c615432f6d3976b7d3e591d0a96a 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-07T10:47:03,829 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1dd5c615432f6d3976b7d3e591d0a96a/.tmp/colfamily11/5b51765e2b3b483eaa759387f861380d is 52, key is www/colfamily11:colfamily11/1730976236620/Put/seqid=0 2024-11-07T10:47:03,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741894_1070 (size=39263) 2024-11-07T10:47:03,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741894_1070 (size=39263) 2024-11-07T10:47:03,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741894_1070 (size=39263) 2024-11-07T10:47:03,838 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1dd5c615432f6d3976b7d3e591d0a96a/.tmp/colfamily11/5b51765e2b3b483eaa759387f861380d 2024-11-07T10:47:03,846 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1dd5c615432f6d3976b7d3e591d0a96a/.tmp/colfamily11/5b51765e2b3b483eaa759387f861380d as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1dd5c615432f6d3976b7d3e591d0a96a/colfamily11/5b51765e2b3b483eaa759387f861380d 2024-11-07T10:47:03,853 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1dd5c615432f6d3976b7d3e591d0a96a/colfamily11/5b51765e2b3b483eaa759387f861380d, entries=703, sequenceid=5, filesize=38.3 K 2024-11-07T10:47:03,854 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 1dd5c615432f6d3976b7d3e591d0a96a in 48ms, sequenceid=5, compaction requested=false 2024-11-07T10:47:03,859 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1dd5c615432f6d3976b7d3e591d0a96a/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-07T10:47:03,860 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1730976232769.1dd5c615432f6d3976b7d3e591d0a96a. 2024-11-07T10:47:03,860 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1dd5c615432f6d3976b7d3e591d0a96a: Waiting for close lock at 1730976423806Running coprocessor pre-close hooks at 1730976423806Disabling compacts and flushes for region at 1730976423806Disabling writes for close at 1730976423806Obtaining lock to block concurrent updates at 1730976423806Preparing flush snapshotting stores in 1dd5c615432f6d3976b7d3e591d0a96a at 1730976423806Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1730976232769.1dd5c615432f6d3976b7d3e591d0a96a., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1730976423806Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1730976232769.1dd5c615432f6d3976b7d3e591d0a96a. at 1730976423807 (+1 ms)Flushing 1dd5c615432f6d3976b7d3e591d0a96a/colfamily11: creating writer at 1730976423807Flushing 1dd5c615432f6d3976b7d3e591d0a96a/colfamily11: appending metadata at 1730976423829 (+22 ms)Flushing 1dd5c615432f6d3976b7d3e591d0a96a/colfamily11: closing flushed file at 1730976423829Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2142e6cc: reopening flushed file at 1730976423845 (+16 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 1dd5c615432f6d3976b7d3e591d0a96a in 48ms, sequenceid=5, compaction requested=false at 1730976423854 (+9 ms)Writing region close event to WAL at 1730976423855 (+1 ms)Running coprocessor post-close hooks at 1730976423860 (+5 ms)Closed at 1730976423860 2024-11-07T10:47:03,860 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1730976232769.1dd5c615432f6d3976b7d3e591d0a96a. 2024-11-07T10:47:03,860 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8f449585c6b7b82f4442604100a49644, disabling compactions & flushes 2024-11-07T10:47:03,860 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644. 2024-11-07T10:47:03,860 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644. 2024-11-07T10:47:03,860 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644. after waiting 0 ms 2024-11-07T10:47:03,860 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644. 2024-11-07T10:47:03,862 DEBUG [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(1351): Waiting on 0b29f5d988381aa77506fa3b92944854, 1baabb362104b46ecb78fe236391ca6f, 3f6021bbda5b8b7b00318a57f02cd77d, 541874b6b96014b10121ede508f1f61f, 72bf61b738a2f9b35913a192fc32815b, d4b381fbb1d4cd388f45774fca53b2eb, dbe69e720cc4f2a4a4bc41171b6b3c32, dec39ef6d94f3a519151705fce17adc4, f89575b73864cac8b7b164f417ef378f 2024-11-07T10:47:03,865 DEBUG [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(1351): Waiting on 0a7d18696a90cfed5af1fbdb2a706b65, 7a68cc5b6ea7d9e2214ab42d65b16eba, 8f449585c6b7b82f4442604100a49644, 9b830813c6b6c3ebcdb59061af10faad, c3ab41a5354d389521b694050824788e, c71057666e459779c2fa5a4311f4e99c, db89a463448eb182960960c83f4e0b16, ddeeab6f2e44d85eb1ee916d3c5e8e9f 2024-11-07T10:47:03,865 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8f449585c6b7b82f4442604100a49644/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-07T10:47:03,866 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644. 2024-11-07T10:47:03,866 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8f449585c6b7b82f4442604100a49644: Waiting for close lock at 1730976423860Running coprocessor pre-close hooks at 1730976423860Disabling compacts and flushes for region at 1730976423860Disabling writes for close at 1730976423860Writing region close event to WAL at 1730976423861 (+1 ms)Running coprocessor post-close hooks at 1730976423866 (+5 ms)Closed at 1730976423866 2024-11-07T10:47:03,866 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1730976232769.8f449585c6b7b82f4442604100a49644. 2024-11-07T10:47:03,866 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing db89a463448eb182960960c83f4e0b16, disabling compactions & flushes 2024-11-07T10:47:03,866 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16. 2024-11-07T10:47:03,867 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16. 2024-11-07T10:47:03,867 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16. after waiting 0 ms 2024-11-07T10:47:03,867 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16. 2024-11-07T10:47:03,870 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/db89a463448eb182960960c83f4e0b16/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-07T10:47:03,871 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16. 2024-11-07T10:47:03,871 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for db89a463448eb182960960c83f4e0b16: Waiting for close lock at 1730976423866Running coprocessor pre-close hooks at 1730976423866Disabling compacts and flushes for region at 1730976423866Disabling writes for close at 1730976423867 (+1 ms)Writing region close event to WAL at 1730976423867Running coprocessor post-close hooks at 1730976423871 (+4 ms)Closed at 1730976423871 2024-11-07T10:47:03,871 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1730976232769.db89a463448eb182960960c83f4e0b16. 2024-11-07T10:47:03,871 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7a68cc5b6ea7d9e2214ab42d65b16eba, disabling compactions & flushes 2024-11-07T10:47:03,871 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba. 2024-11-07T10:47:03,871 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba. 2024-11-07T10:47:03,871 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba. after waiting 0 ms 2024-11-07T10:47:03,871 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba. 2024-11-07T10:47:03,877 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/7a68cc5b6ea7d9e2214ab42d65b16eba/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-07T10:47:03,877 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba. 2024-11-07T10:47:03,877 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7a68cc5b6ea7d9e2214ab42d65b16eba: Waiting for close lock at 1730976423871Running coprocessor pre-close hooks at 1730976423871Disabling compacts and flushes for region at 1730976423871Disabling writes for close at 1730976423871Writing region close event to WAL at 1730976423873 (+2 ms)Running coprocessor post-close hooks at 1730976423877 (+4 ms)Closed at 1730976423877 2024-11-07T10:47:03,877 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1730976232769.7a68cc5b6ea7d9e2214ab42d65b16eba. 2024-11-07T10:47:03,878 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ddeeab6f2e44d85eb1ee916d3c5e8e9f, disabling compactions & flushes 2024-11-07T10:47:03,878 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1730976232769.ddeeab6f2e44d85eb1ee916d3c5e8e9f. 2024-11-07T10:47:03,878 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1730976232769.ddeeab6f2e44d85eb1ee916d3c5e8e9f. 2024-11-07T10:47:03,878 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1730976232769.ddeeab6f2e44d85eb1ee916d3c5e8e9f. after waiting 0 ms 2024-11-07T10:47:03,878 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1730976232769.ddeeab6f2e44d85eb1ee916d3c5e8e9f. 2024-11-07T10:47:03,878 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing ddeeab6f2e44d85eb1ee916d3c5e8e9f 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-07T10:47:03,901 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ddeeab6f2e44d85eb1ee916d3c5e8e9f/.tmp/colfamily11/25ea6cce25bc4e9fa6c1a37da9d6001b is 52, key is fff/colfamily11:colfamily11/1730976236321/Put/seqid=0 2024-11-07T10:47:03,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741895_1071 (size=39263) 2024-11-07T10:47:03,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741895_1071 (size=39263) 2024-11-07T10:47:03,910 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ddeeab6f2e44d85eb1ee916d3c5e8e9f/.tmp/colfamily11/25ea6cce25bc4e9fa6c1a37da9d6001b 2024-11-07T10:47:03,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741895_1071 (size=39263) 2024-11-07T10:47:03,916 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ddeeab6f2e44d85eb1ee916d3c5e8e9f/.tmp/colfamily11/25ea6cce25bc4e9fa6c1a37da9d6001b as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ddeeab6f2e44d85eb1ee916d3c5e8e9f/colfamily11/25ea6cce25bc4e9fa6c1a37da9d6001b 2024-11-07T10:47:03,922 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ddeeab6f2e44d85eb1ee916d3c5e8e9f/colfamily11/25ea6cce25bc4e9fa6c1a37da9d6001b, entries=703, sequenceid=5, filesize=38.3 K 2024-11-07T10:47:03,923 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for ddeeab6f2e44d85eb1ee916d3c5e8e9f in 45ms, sequenceid=5, compaction requested=false 2024-11-07T10:47:03,927 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ddeeab6f2e44d85eb1ee916d3c5e8e9f/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-07T10:47:03,927 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1730976232769.ddeeab6f2e44d85eb1ee916d3c5e8e9f. 2024-11-07T10:47:03,927 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ddeeab6f2e44d85eb1ee916d3c5e8e9f: Waiting for close lock at 1730976423878Running coprocessor pre-close hooks at 1730976423878Disabling compacts and flushes for region at 1730976423878Disabling writes for close at 1730976423878Obtaining lock to block concurrent updates at 1730976423878Preparing flush snapshotting stores in ddeeab6f2e44d85eb1ee916d3c5e8e9f at 1730976423878Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1730976232769.ddeeab6f2e44d85eb1ee916d3c5e8e9f., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1730976423878Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1730976232769.ddeeab6f2e44d85eb1ee916d3c5e8e9f. at 1730976423879 (+1 ms)Flushing ddeeab6f2e44d85eb1ee916d3c5e8e9f/colfamily11: creating writer at 1730976423879Flushing ddeeab6f2e44d85eb1ee916d3c5e8e9f/colfamily11: appending metadata at 1730976423901 (+22 ms)Flushing ddeeab6f2e44d85eb1ee916d3c5e8e9f/colfamily11: closing flushed file at 1730976423901Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e395411: reopening flushed file at 1730976423915 (+14 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for ddeeab6f2e44d85eb1ee916d3c5e8e9f in 45ms, sequenceid=5, compaction requested=false at 1730976423923 (+8 ms)Writing region close event to WAL at 1730976423924 (+1 ms)Running coprocessor post-close hooks at 1730976423927 (+3 ms)Closed at 1730976423927 2024-11-07T10:47:03,927 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1730976232769.ddeeab6f2e44d85eb1ee916d3c5e8e9f. 2024-11-07T10:47:03,927 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9b830813c6b6c3ebcdb59061af10faad, disabling compactions & flushes 2024-11-07T10:47:03,928 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1730976232769.9b830813c6b6c3ebcdb59061af10faad. 2024-11-07T10:47:03,928 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1730976232769.9b830813c6b6c3ebcdb59061af10faad. 2024-11-07T10:47:03,928 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1730976232769.9b830813c6b6c3ebcdb59061af10faad. after waiting 0 ms 2024-11-07T10:47:03,928 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1730976232769.9b830813c6b6c3ebcdb59061af10faad. 2024-11-07T10:47:03,928 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 9b830813c6b6c3ebcdb59061af10faad 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-07T10:47:03,944 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9b830813c6b6c3ebcdb59061af10faad/.tmp/colfamily11/0321993bd1a54aecbc6b23c0fc04151e is 52, key is nnn/colfamily11:colfamily11/1730976236494/Put/seqid=0 2024-11-07T10:47:03,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741896_1072 (size=39263) 2024-11-07T10:47:03,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741896_1072 (size=39263) 2024-11-07T10:47:03,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741896_1072 (size=39263) 2024-11-07T10:47:03,956 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9b830813c6b6c3ebcdb59061af10faad/.tmp/colfamily11/0321993bd1a54aecbc6b23c0fc04151e 2024-11-07T10:47:03,964 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9b830813c6b6c3ebcdb59061af10faad/.tmp/colfamily11/0321993bd1a54aecbc6b23c0fc04151e as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9b830813c6b6c3ebcdb59061af10faad/colfamily11/0321993bd1a54aecbc6b23c0fc04151e 2024-11-07T10:47:03,970 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9b830813c6b6c3ebcdb59061af10faad/colfamily11/0321993bd1a54aecbc6b23c0fc04151e, entries=703, sequenceid=5, filesize=38.3 K 2024-11-07T10:47:03,971 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 9b830813c6b6c3ebcdb59061af10faad in 43ms, sequenceid=5, compaction requested=false 2024-11-07T10:47:03,975 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9b830813c6b6c3ebcdb59061af10faad/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-07T10:47:03,976 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1730976232769.9b830813c6b6c3ebcdb59061af10faad. 2024-11-07T10:47:03,976 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9b830813c6b6c3ebcdb59061af10faad: Waiting for close lock at 1730976423927Running coprocessor pre-close hooks at 1730976423927Disabling compacts and flushes for region at 1730976423927Disabling writes for close at 1730976423928 (+1 ms)Obtaining lock to block concurrent updates at 1730976423928Preparing flush snapshotting stores in 9b830813c6b6c3ebcdb59061af10faad at 1730976423928Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1730976232769.9b830813c6b6c3ebcdb59061af10faad., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1730976423928Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1730976232769.9b830813c6b6c3ebcdb59061af10faad. at 1730976423928Flushing 9b830813c6b6c3ebcdb59061af10faad/colfamily11: creating writer at 1730976423929 (+1 ms)Flushing 9b830813c6b6c3ebcdb59061af10faad/colfamily11: appending metadata at 1730976423944 (+15 ms)Flushing 9b830813c6b6c3ebcdb59061af10faad/colfamily11: closing flushed file at 1730976423944Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34ad7673: reopening flushed file at 1730976423963 (+19 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 9b830813c6b6c3ebcdb59061af10faad in 43ms, sequenceid=5, compaction requested=false at 1730976423971 (+8 ms)Writing region close event to WAL at 1730976423972 (+1 ms)Running coprocessor post-close hooks at 1730976423976 (+4 ms)Closed at 1730976423976 2024-11-07T10:47:03,976 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1730976232769.9b830813c6b6c3ebcdb59061af10faad. 2024-11-07T10:47:03,976 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c71057666e459779c2fa5a4311f4e99c, disabling compactions & flushes 2024-11-07T10:47:03,976 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1730976232769.c71057666e459779c2fa5a4311f4e99c. 2024-11-07T10:47:03,976 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1730976232769.c71057666e459779c2fa5a4311f4e99c. 2024-11-07T10:47:03,976 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1730976232769.c71057666e459779c2fa5a4311f4e99c. after waiting 0 ms 2024-11-07T10:47:03,976 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1730976232769.c71057666e459779c2fa5a4311f4e99c. 2024-11-07T10:47:03,976 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing c71057666e459779c2fa5a4311f4e99c 3/3 column families, dataSize=48 B heapSize=888 B 2024-11-07T10:47:03,998 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c71057666e459779c2fa5a4311f4e99c/.tmp/colfamily11/253d392d9549404cbbc90b93ff66e5e0 is 52, key is zzz/colfamily11:colfamily11/1730976236628/Put/seqid=0 2024-11-07T10:47:04,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741897_1073 (size=4857) 2024-11-07T10:47:04,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741897_1073 (size=4857) 2024-11-07T10:47:04,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741897_1073 (size=4857) 2024-11-07T10:47:04,006 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c71057666e459779c2fa5a4311f4e99c/.tmp/colfamily11/253d392d9549404cbbc90b93ff66e5e0 2024-11-07T10:47:04,013 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c71057666e459779c2fa5a4311f4e99c/.tmp/colfamily11/253d392d9549404cbbc90b93ff66e5e0 as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c71057666e459779c2fa5a4311f4e99c/colfamily11/253d392d9549404cbbc90b93ff66e5e0 2024-11-07T10:47:04,019 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c71057666e459779c2fa5a4311f4e99c/colfamily11/253d392d9549404cbbc90b93ff66e5e0, entries=1, sequenceid=5, filesize=4.7 K 2024-11-07T10:47:04,020 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~48 B/48, heapSize ~360 B/360, currentSize=0 B/0 for c71057666e459779c2fa5a4311f4e99c in 44ms, sequenceid=5, compaction requested=false 2024-11-07T10:47:04,024 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c71057666e459779c2fa5a4311f4e99c/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-07T10:47:04,025 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1730976232769.c71057666e459779c2fa5a4311f4e99c. 2024-11-07T10:47:04,025 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c71057666e459779c2fa5a4311f4e99c: Waiting for close lock at 1730976423976Running coprocessor pre-close hooks at 1730976423976Disabling compacts and flushes for region at 1730976423976Disabling writes for close at 1730976423976Obtaining lock to block concurrent updates at 1730976423976Preparing flush snapshotting stores in c71057666e459779c2fa5a4311f4e99c at 1730976423976Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1730976232769.c71057666e459779c2fa5a4311f4e99c., syncing WAL and waiting on mvcc, flushsize=dataSize=48, getHeapSize=840, getOffHeapSize=0, getCellsCount=1 at 1730976423977 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1730976232769.c71057666e459779c2fa5a4311f4e99c. at 1730976423977Flushing c71057666e459779c2fa5a4311f4e99c/colfamily11: creating writer at 1730976423977Flushing c71057666e459779c2fa5a4311f4e99c/colfamily11: appending metadata at 1730976423998 (+21 ms)Flushing c71057666e459779c2fa5a4311f4e99c/colfamily11: closing flushed file at 1730976423998Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3574d5fa: reopening flushed file at 1730976424012 (+14 ms)Finished flush of dataSize ~48 B/48, heapSize ~360 B/360, currentSize=0 B/0 for c71057666e459779c2fa5a4311f4e99c in 44ms, sequenceid=5, compaction requested=false at 1730976424020 (+8 ms)Writing region close event to WAL at 1730976424021 (+1 ms)Running coprocessor post-close hooks at 1730976424025 (+4 ms)Closed at 1730976424025 2024-11-07T10:47:04,025 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1730976232769.c71057666e459779c2fa5a4311f4e99c. 2024-11-07T10:47:04,025 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0a7d18696a90cfed5af1fbdb2a706b65, disabling compactions & flushes 2024-11-07T10:47:04,025 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1730976232769.0a7d18696a90cfed5af1fbdb2a706b65. 2024-11-07T10:47:04,025 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1730976232769.0a7d18696a90cfed5af1fbdb2a706b65. 2024-11-07T10:47:04,025 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1730976232769.0a7d18696a90cfed5af1fbdb2a706b65. after waiting 0 ms 2024-11-07T10:47:04,025 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1730976232769.0a7d18696a90cfed5af1fbdb2a706b65. 2024-11-07T10:47:04,025 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 0a7d18696a90cfed5af1fbdb2a706b65 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-07T10:47:04,043 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0a7d18696a90cfed5af1fbdb2a706b65/.tmp/colfamily11/0c330fa82e9a400b9ae3965c66d11c18 is 52, key is kkk/colfamily11:colfamily11/1730976236443/Put/seqid=0 2024-11-07T10:47:04,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741898_1074 (size=39263) 2024-11-07T10:47:04,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741898_1074 (size=39263) 2024-11-07T10:47:04,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741898_1074 (size=39263) 2024-11-07T10:47:04,053 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0a7d18696a90cfed5af1fbdb2a706b65/.tmp/colfamily11/0c330fa82e9a400b9ae3965c66d11c18 2024-11-07T10:47:04,061 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0a7d18696a90cfed5af1fbdb2a706b65/.tmp/colfamily11/0c330fa82e9a400b9ae3965c66d11c18 as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0a7d18696a90cfed5af1fbdb2a706b65/colfamily11/0c330fa82e9a400b9ae3965c66d11c18 2024-11-07T10:47:04,062 DEBUG [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(1351): Waiting on 0b29f5d988381aa77506fa3b92944854, 1baabb362104b46ecb78fe236391ca6f, 3f6021bbda5b8b7b00318a57f02cd77d, 541874b6b96014b10121ede508f1f61f, 72bf61b738a2f9b35913a192fc32815b, d4b381fbb1d4cd388f45774fca53b2eb, dbe69e720cc4f2a4a4bc41171b6b3c32, dec39ef6d94f3a519151705fce17adc4, f89575b73864cac8b7b164f417ef378f 2024-11-07T10:47:04,065 DEBUG [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(1351): Waiting on 0a7d18696a90cfed5af1fbdb2a706b65, c3ab41a5354d389521b694050824788e 2024-11-07T10:47:04,067 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0a7d18696a90cfed5af1fbdb2a706b65/colfamily11/0c330fa82e9a400b9ae3965c66d11c18, entries=703, sequenceid=5, filesize=38.3 K 2024-11-07T10:47:04,068 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 0a7d18696a90cfed5af1fbdb2a706b65 in 43ms, sequenceid=5, compaction requested=false 2024-11-07T10:47:04,076 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0a7d18696a90cfed5af1fbdb2a706b65/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-07T10:47:04,077 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1730976232769.0a7d18696a90cfed5af1fbdb2a706b65. 2024-11-07T10:47:04,077 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0a7d18696a90cfed5af1fbdb2a706b65: Waiting for close lock at 1730976424025Running coprocessor pre-close hooks at 1730976424025Disabling compacts and flushes for region at 1730976424025Disabling writes for close at 1730976424025Obtaining lock to block concurrent updates at 1730976424025Preparing flush snapshotting stores in 0a7d18696a90cfed5af1fbdb2a706b65 at 1730976424025Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1730976232769.0a7d18696a90cfed5af1fbdb2a706b65., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1730976424026 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1730976232769.0a7d18696a90cfed5af1fbdb2a706b65. at 1730976424028 (+2 ms)Flushing 0a7d18696a90cfed5af1fbdb2a706b65/colfamily11: creating writer at 1730976424028Flushing 0a7d18696a90cfed5af1fbdb2a706b65/colfamily11: appending metadata at 1730976424043 (+15 ms)Flushing 0a7d18696a90cfed5af1fbdb2a706b65/colfamily11: closing flushed file at 1730976424043Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38f44: reopening flushed file at 1730976424060 (+17 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 0a7d18696a90cfed5af1fbdb2a706b65 in 43ms, sequenceid=5, compaction requested=false at 1730976424068 (+8 ms)Writing region close event to WAL at 1730976424070 (+2 ms)Running coprocessor post-close hooks at 1730976424077 (+7 ms)Closed at 1730976424077 2024-11-07T10:47:04,077 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1730976232769.0a7d18696a90cfed5af1fbdb2a706b65. 2024-11-07T10:47:04,077 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c3ab41a5354d389521b694050824788e, disabling compactions & flushes 2024-11-07T10:47:04,077 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1730976232769.c3ab41a5354d389521b694050824788e. 2024-11-07T10:47:04,077 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1730976232769.c3ab41a5354d389521b694050824788e. 2024-11-07T10:47:04,077 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1730976232769.c3ab41a5354d389521b694050824788e. after waiting 0 ms 2024-11-07T10:47:04,077 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1730976232769.c3ab41a5354d389521b694050824788e. 2024-11-07T10:47:04,078 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing c3ab41a5354d389521b694050824788e 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-07T10:47:04,101 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3ab41a5354d389521b694050824788e/.tmp/colfamily11/b10bbafc0ce641b4b0aea5519c5ea351 is 52, key is hhh/colfamily11:colfamily11/1730976236400/Put/seqid=0 2024-11-07T10:47:04,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741899_1075 (size=39263) 2024-11-07T10:47:04,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741899_1075 (size=39263) 2024-11-07T10:47:04,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741899_1075 (size=39263) 2024-11-07T10:47:04,112 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3ab41a5354d389521b694050824788e/.tmp/colfamily11/b10bbafc0ce641b4b0aea5519c5ea351 2024-11-07T10:47:04,119 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3ab41a5354d389521b694050824788e/.tmp/colfamily11/b10bbafc0ce641b4b0aea5519c5ea351 as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3ab41a5354d389521b694050824788e/colfamily11/b10bbafc0ce641b4b0aea5519c5ea351 2024-11-07T10:47:04,125 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3ab41a5354d389521b694050824788e/colfamily11/b10bbafc0ce641b4b0aea5519c5ea351, entries=703, sequenceid=5, filesize=38.3 K 2024-11-07T10:47:04,127 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for c3ab41a5354d389521b694050824788e in 49ms, sequenceid=5, compaction requested=false 2024-11-07T10:47:04,130 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c3ab41a5354d389521b694050824788e/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-07T10:47:04,131 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1730976232769.c3ab41a5354d389521b694050824788e. 2024-11-07T10:47:04,131 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c3ab41a5354d389521b694050824788e: Waiting for close lock at 1730976424077Running coprocessor pre-close hooks at 1730976424077Disabling compacts and flushes for region at 1730976424077Disabling writes for close at 1730976424077Obtaining lock to block concurrent updates at 1730976424078 (+1 ms)Preparing flush snapshotting stores in c3ab41a5354d389521b694050824788e at 1730976424078Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1730976232769.c3ab41a5354d389521b694050824788e., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1730976424078Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1730976232769.c3ab41a5354d389521b694050824788e. at 1730976424078Flushing c3ab41a5354d389521b694050824788e/colfamily11: creating writer at 1730976424079 (+1 ms)Flushing c3ab41a5354d389521b694050824788e/colfamily11: appending metadata at 1730976424101 (+22 ms)Flushing c3ab41a5354d389521b694050824788e/colfamily11: closing flushed file at 1730976424101Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58f8aff0: reopening flushed file at 1730976424118 (+17 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for c3ab41a5354d389521b694050824788e in 49ms, sequenceid=5, compaction requested=false at 1730976424127 (+9 ms)Writing region close event to WAL at 1730976424127Running coprocessor post-close hooks at 1730976424131 (+4 ms)Closed at 1730976424131 2024-11-07T10:47:04,131 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1730976232769.c3ab41a5354d389521b694050824788e. 2024-11-07T10:47:04,188 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1baabb362104b46ecb78fe236391ca6f/.tmp/colfamily11/44c57c7514b84fd6a5d3fa810f4b7149 2024-11-07T10:47:04,201 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1baabb362104b46ecb78fe236391ca6f/.tmp/colfamily11/44c57c7514b84fd6a5d3fa810f4b7149 as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1baabb362104b46ecb78fe236391ca6f/colfamily11/44c57c7514b84fd6a5d3fa810f4b7149 2024-11-07T10:47:04,209 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1baabb362104b46ecb78fe236391ca6f/colfamily11/44c57c7514b84fd6a5d3fa810f4b7149, entries=703, sequenceid=5, filesize=38.3 K 2024-11-07T10:47:04,210 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 1baabb362104b46ecb78fe236391ca6f in 488ms, sequenceid=5, compaction requested=false 2024-11-07T10:47:04,214 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/1baabb362104b46ecb78fe236391ca6f/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-07T10:47:04,215 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1730976232769.1baabb362104b46ecb78fe236391ca6f. 2024-11-07T10:47:04,215 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1baabb362104b46ecb78fe236391ca6f: Waiting for close lock at 1730976423722Running coprocessor pre-close hooks at 1730976423722Disabling compacts and flushes for region at 1730976423722Disabling writes for close at 1730976423722Obtaining lock to block concurrent updates at 1730976423722Preparing flush snapshotting stores in 1baabb362104b46ecb78fe236391ca6f at 1730976423722Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1730976232769.1baabb362104b46ecb78fe236391ca6f., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1730976423723 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1730976232769.1baabb362104b46ecb78fe236391ca6f. at 1730976423723Flushing 1baabb362104b46ecb78fe236391ca6f/colfamily11: creating writer at 1730976423723Flushing 1baabb362104b46ecb78fe236391ca6f/colfamily11: appending metadata at 1730976423755 (+32 ms)Flushing 1baabb362104b46ecb78fe236391ca6f/colfamily11: closing flushed file at 1730976423755Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6dc7acec: reopening flushed file at 1730976424199 (+444 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 1baabb362104b46ecb78fe236391ca6f in 488ms, sequenceid=5, compaction requested=false at 1730976424210 (+11 ms)Writing region close event to WAL at 1730976424211 (+1 ms)Running coprocessor post-close hooks at 1730976424215 (+4 ms)Closed at 1730976424215 2024-11-07T10:47:04,215 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1730976232769.1baabb362104b46ecb78fe236391ca6f. 2024-11-07T10:47:04,215 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d4b381fbb1d4cd388f45774fca53b2eb, disabling compactions & flushes 2024-11-07T10:47:04,215 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb. 2024-11-07T10:47:04,215 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb. 2024-11-07T10:47:04,216 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb. after waiting 0 ms 2024-11-07T10:47:04,216 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb. 2024-11-07T10:47:04,220 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d4b381fbb1d4cd388f45774fca53b2eb/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-07T10:47:04,220 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb. 2024-11-07T10:47:04,221 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d4b381fbb1d4cd388f45774fca53b2eb: Waiting for close lock at 1730976424215Running coprocessor pre-close hooks at 1730976424215Disabling compacts and flushes for region at 1730976424215Disabling writes for close at 1730976424216 (+1 ms)Writing region close event to WAL at 1730976424217 (+1 ms)Running coprocessor post-close hooks at 1730976424220 (+3 ms)Closed at 1730976424220 2024-11-07T10:47:04,221 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1730976232769.d4b381fbb1d4cd388f45774fca53b2eb. 2024-11-07T10:47:04,221 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dbe69e720cc4f2a4a4bc41171b6b3c32, disabling compactions & flushes 2024-11-07T10:47:04,221 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1730976232769.dbe69e720cc4f2a4a4bc41171b6b3c32. 2024-11-07T10:47:04,221 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1730976232769.dbe69e720cc4f2a4a4bc41171b6b3c32. 2024-11-07T10:47:04,221 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1730976232769.dbe69e720cc4f2a4a4bc41171b6b3c32. after waiting 0 ms 2024-11-07T10:47:04,221 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1730976232769.dbe69e720cc4f2a4a4bc41171b6b3c32. 2024-11-07T10:47:04,221 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing dbe69e720cc4f2a4a4bc41171b6b3c32 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-07T10:47:04,241 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dbe69e720cc4f2a4a4bc41171b6b3c32/.tmp/colfamily11/2087c03a27624372851c9fc5c8f62502 is 52, key is ccc/colfamily11:colfamily11/1730976236568/Put/seqid=0 2024-11-07T10:47:04,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741900_1076 (size=39263) 2024-11-07T10:47:04,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741900_1076 (size=39263) 2024-11-07T10:47:04,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741900_1076 (size=39263) 2024-11-07T10:47:04,249 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dbe69e720cc4f2a4a4bc41171b6b3c32/.tmp/colfamily11/2087c03a27624372851c9fc5c8f62502 2024-11-07T10:47:04,255 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dbe69e720cc4f2a4a4bc41171b6b3c32/.tmp/colfamily11/2087c03a27624372851c9fc5c8f62502 as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dbe69e720cc4f2a4a4bc41171b6b3c32/colfamily11/2087c03a27624372851c9fc5c8f62502 2024-11-07T10:47:04,260 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dbe69e720cc4f2a4a4bc41171b6b3c32/colfamily11/2087c03a27624372851c9fc5c8f62502, entries=703, sequenceid=5, filesize=38.3 K 2024-11-07T10:47:04,261 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for dbe69e720cc4f2a4a4bc41171b6b3c32 in 40ms, sequenceid=5, compaction requested=false 2024-11-07T10:47:04,262 DEBUG [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(1351): Waiting on 0b29f5d988381aa77506fa3b92944854, 3f6021bbda5b8b7b00318a57f02cd77d, 541874b6b96014b10121ede508f1f61f, 72bf61b738a2f9b35913a192fc32815b, dbe69e720cc4f2a4a4bc41171b6b3c32, dec39ef6d94f3a519151705fce17adc4, f89575b73864cac8b7b164f417ef378f 2024-11-07T10:47:04,265 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dbe69e720cc4f2a4a4bc41171b6b3c32/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-07T10:47:04,265 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1730976232769.dbe69e720cc4f2a4a4bc41171b6b3c32. 2024-11-07T10:47:04,265 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(976): stopping server 10641b2c0a69,40573,1730976230476; all regions closed. 2024-11-07T10:47:04,265 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dbe69e720cc4f2a4a4bc41171b6b3c32: Waiting for close lock at 1730976424221Running coprocessor pre-close hooks at 1730976424221Disabling compacts and flushes for region at 1730976424221Disabling writes for close at 1730976424221Obtaining lock to block concurrent updates at 1730976424221Preparing flush snapshotting stores in dbe69e720cc4f2a4a4bc41171b6b3c32 at 1730976424221Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1730976232769.dbe69e720cc4f2a4a4bc41171b6b3c32., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1730976424221Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1730976232769.dbe69e720cc4f2a4a4bc41171b6b3c32. at 1730976424222 (+1 ms)Flushing dbe69e720cc4f2a4a4bc41171b6b3c32/colfamily11: creating writer at 1730976424222Flushing dbe69e720cc4f2a4a4bc41171b6b3c32/colfamily11: appending metadata at 1730976424241 (+19 ms)Flushing dbe69e720cc4f2a4a4bc41171b6b3c32/colfamily11: closing flushed file at 1730976424241Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@402a8bf2: reopening flushed file at 1730976424254 (+13 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for dbe69e720cc4f2a4a4bc41171b6b3c32 in 40ms, sequenceid=5, compaction requested=false at 1730976424261 (+7 ms)Writing region close event to WAL at 1730976424262 (+1 ms)Running coprocessor post-close hooks at 1730976424265 (+3 ms)Closed at 1730976424265 2024-11-07T10:47:04,266 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1730976232769.dbe69e720cc4f2a4a4bc41171b6b3c32. 2024-11-07T10:47:04,266 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dec39ef6d94f3a519151705fce17adc4, disabling compactions & flushes 2024-11-07T10:47:04,266 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1730976232769.dec39ef6d94f3a519151705fce17adc4. 2024-11-07T10:47:04,266 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1730976232769.dec39ef6d94f3a519151705fce17adc4. 2024-11-07T10:47:04,266 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1730976232769.dec39ef6d94f3a519151705fce17adc4. after waiting 0 ms 2024-11-07T10:47:04,266 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1730976232769.dec39ef6d94f3a519151705fce17adc4. 2024-11-07T10:47:04,266 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing dec39ef6d94f3a519151705fce17adc4 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-07T10:47:04,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741865_1041 (size=15722) 2024-11-07T10:47:04,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741865_1041 (size=15722) 2024-11-07T10:47:04,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741865_1041 (size=15722) 2024-11-07T10:47:04,271 DEBUG [RS:0;10641b2c0a69:40573 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/oldWALs 2024-11-07T10:47:04,271 INFO [RS:0;10641b2c0a69:40573 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 10641b2c0a69%2C40573%2C1730976230476.meta:.meta(num 1730976418024) 2024-11-07T10:47:04,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741834_1010 (size=317622) 2024-11-07T10:47:04,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741834_1010 (size=317622) 2024-11-07T10:47:04,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741834_1010 (size=317622) 2024-11-07T10:47:04,285 DEBUG [RS:0;10641b2c0a69:40573 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/oldWALs 2024-11-07T10:47:04,285 INFO [RS:0;10641b2c0a69:40573 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 10641b2c0a69%2C40573%2C1730976230476:(num 1730976231927) 2024-11-07T10:47:04,285 DEBUG [RS:0;10641b2c0a69:40573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T10:47:04,285 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T10:47:04,285 INFO [RS:0;10641b2c0a69:40573 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T10:47:04,286 INFO [RS:0;10641b2c0a69:40573 {}] hbase.ChoreService(370): Chore service for: regionserver/10641b2c0a69:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-07T10:47:04,286 INFO [RS:0;10641b2c0a69:40573 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T10:47:04,286 INFO [regionserver/10641b2c0a69:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T10:47:04,286 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dec39ef6d94f3a519151705fce17adc4/.tmp/colfamily11/eda1bc90f5ed4d7796a8fd1895c2ce62 is 52, key is ooo/colfamily11:colfamily11/1730976236736/Put/seqid=0 2024-11-07T10:47:04,286 INFO [RS:0;10641b2c0a69:40573 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40573 2024-11-07T10:47:04,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/10641b2c0a69,40573,1730976230476 2024-11-07T10:47:04,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T10:47:04,289 INFO [RS:0;10641b2c0a69:40573 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T10:47:04,290 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [10641b2c0a69,40573,1730976230476] 2024-11-07T10:47:04,291 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/10641b2c0a69,40573,1730976230476 already deleted, retry=false 2024-11-07T10:47:04,292 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 10641b2c0a69,40573,1730976230476 expired; onlineServers=1 2024-11-07T10:47:04,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741901_1077 (size=39263) 2024-11-07T10:47:04,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741901_1077 (size=39263) 2024-11-07T10:47:04,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741901_1077 (size=39263) 2024-11-07T10:47:04,303 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dec39ef6d94f3a519151705fce17adc4/.tmp/colfamily11/eda1bc90f5ed4d7796a8fd1895c2ce62 2024-11-07T10:47:04,309 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dec39ef6d94f3a519151705fce17adc4/.tmp/colfamily11/eda1bc90f5ed4d7796a8fd1895c2ce62 as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dec39ef6d94f3a519151705fce17adc4/colfamily11/eda1bc90f5ed4d7796a8fd1895c2ce62 2024-11-07T10:47:04,315 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dec39ef6d94f3a519151705fce17adc4/colfamily11/eda1bc90f5ed4d7796a8fd1895c2ce62, entries=703, sequenceid=5, filesize=38.3 K 2024-11-07T10:47:04,316 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for dec39ef6d94f3a519151705fce17adc4 in 50ms, sequenceid=5, compaction requested=false 2024-11-07T10:47:04,319 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/dec39ef6d94f3a519151705fce17adc4/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-07T10:47:04,320 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1730976232769.dec39ef6d94f3a519151705fce17adc4. 2024-11-07T10:47:04,320 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dec39ef6d94f3a519151705fce17adc4: Waiting for close lock at 1730976424266Running coprocessor pre-close hooks at 1730976424266Disabling compacts and flushes for region at 1730976424266Disabling writes for close at 1730976424266Obtaining lock to block concurrent updates at 1730976424266Preparing flush snapshotting stores in dec39ef6d94f3a519151705fce17adc4 at 1730976424266Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1730976232769.dec39ef6d94f3a519151705fce17adc4., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1730976424266Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1730976232769.dec39ef6d94f3a519151705fce17adc4. at 1730976424267 (+1 ms)Flushing dec39ef6d94f3a519151705fce17adc4/colfamily11: creating writer at 1730976424267Flushing dec39ef6d94f3a519151705fce17adc4/colfamily11: appending metadata at 1730976424286 (+19 ms)Flushing dec39ef6d94f3a519151705fce17adc4/colfamily11: closing flushed file at 1730976424286Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c6826fb: reopening flushed file at 1730976424309 (+23 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for dec39ef6d94f3a519151705fce17adc4 in 50ms, sequenceid=5, compaction requested=false at 1730976424316 (+7 ms)Writing region close event to WAL at 1730976424316Running coprocessor post-close hooks at 1730976424320 (+4 ms)Closed at 1730976424320 2024-11-07T10:47:04,320 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1730976232769.dec39ef6d94f3a519151705fce17adc4. 2024-11-07T10:47:04,320 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 72bf61b738a2f9b35913a192fc32815b, disabling compactions & flushes 2024-11-07T10:47:04,320 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b. 2024-11-07T10:47:04,320 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b. 2024-11-07T10:47:04,320 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b. after waiting 0 ms 2024-11-07T10:47:04,320 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b. 2024-11-07T10:47:04,324 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/72bf61b738a2f9b35913a192fc32815b/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-07T10:47:04,325 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b. 2024-11-07T10:47:04,325 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 72bf61b738a2f9b35913a192fc32815b: Waiting for close lock at 1730976424320Running coprocessor pre-close hooks at 1730976424320Disabling compacts and flushes for region at 1730976424320Disabling writes for close at 1730976424320Writing region close event to WAL at 1730976424321 (+1 ms)Running coprocessor post-close hooks at 1730976424325 (+4 ms)Closed at 1730976424325 2024-11-07T10:47:04,325 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1730976232769.72bf61b738a2f9b35913a192fc32815b. 2024-11-07T10:47:04,325 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f89575b73864cac8b7b164f417ef378f, disabling compactions & flushes 2024-11-07T10:47:04,325 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1730976232769.f89575b73864cac8b7b164f417ef378f. 2024-11-07T10:47:04,325 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1730976232769.f89575b73864cac8b7b164f417ef378f. 2024-11-07T10:47:04,325 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1730976232769.f89575b73864cac8b7b164f417ef378f. after waiting 0 ms 2024-11-07T10:47:04,325 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1730976232769.f89575b73864cac8b7b164f417ef378f. 2024-11-07T10:47:04,325 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing f89575b73864cac8b7b164f417ef378f 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-07T10:47:04,347 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f89575b73864cac8b7b164f417ef378f/.tmp/colfamily11/28864dee3cb940be9cbfb4cb8b1b30d3 is 52, key is iii/colfamily11:colfamily11/1730976236654/Put/seqid=0 2024-11-07T10:47:04,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741902_1078 (size=39263) 2024-11-07T10:47:04,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741902_1078 (size=39263) 2024-11-07T10:47:04,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741902_1078 (size=39263) 2024-11-07T10:47:04,360 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f89575b73864cac8b7b164f417ef378f/.tmp/colfamily11/28864dee3cb940be9cbfb4cb8b1b30d3 2024-11-07T10:47:04,367 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f89575b73864cac8b7b164f417ef378f/.tmp/colfamily11/28864dee3cb940be9cbfb4cb8b1b30d3 as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f89575b73864cac8b7b164f417ef378f/colfamily11/28864dee3cb940be9cbfb4cb8b1b30d3 2024-11-07T10:47:04,372 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f89575b73864cac8b7b164f417ef378f/colfamily11/28864dee3cb940be9cbfb4cb8b1b30d3, entries=703, sequenceid=5, filesize=38.3 K 2024-11-07T10:47:04,373 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for f89575b73864cac8b7b164f417ef378f in 48ms, sequenceid=5, compaction requested=false 2024-11-07T10:47:04,377 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f89575b73864cac8b7b164f417ef378f/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-07T10:47:04,378 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1730976232769.f89575b73864cac8b7b164f417ef378f. 2024-11-07T10:47:04,378 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f89575b73864cac8b7b164f417ef378f: Waiting for close lock at 1730976424325Running coprocessor pre-close hooks at 1730976424325Disabling compacts and flushes for region at 1730976424325Disabling writes for close at 1730976424325Obtaining lock to block concurrent updates at 1730976424325Preparing flush snapshotting stores in f89575b73864cac8b7b164f417ef378f at 1730976424325Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1730976232769.f89575b73864cac8b7b164f417ef378f., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1730976424326 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1730976232769.f89575b73864cac8b7b164f417ef378f. at 1730976424327 (+1 ms)Flushing f89575b73864cac8b7b164f417ef378f/colfamily11: creating writer at 1730976424327Flushing f89575b73864cac8b7b164f417ef378f/colfamily11: appending metadata at 1730976424347 (+20 ms)Flushing f89575b73864cac8b7b164f417ef378f/colfamily11: closing flushed file at 1730976424347Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@692e57da: reopening flushed file at 1730976424366 (+19 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for f89575b73864cac8b7b164f417ef378f in 48ms, sequenceid=5, compaction requested=false at 1730976424373 (+7 ms)Writing region close event to WAL at 1730976424374 (+1 ms)Running coprocessor post-close hooks at 1730976424378 (+4 ms)Closed at 1730976424378 2024-11-07T10:47:04,378 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1730976232769.f89575b73864cac8b7b164f417ef378f. 2024-11-07T10:47:04,378 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0b29f5d988381aa77506fa3b92944854, disabling compactions & flushes 2024-11-07T10:47:04,378 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1730976232769.0b29f5d988381aa77506fa3b92944854. 2024-11-07T10:47:04,378 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1730976232769.0b29f5d988381aa77506fa3b92944854. 2024-11-07T10:47:04,378 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1730976232769.0b29f5d988381aa77506fa3b92944854. after waiting 0 ms 2024-11-07T10:47:04,378 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1730976232769.0b29f5d988381aa77506fa3b92944854. 2024-11-07T10:47:04,378 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 0b29f5d988381aa77506fa3b92944854 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-07T10:47:04,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T10:47:04,390 INFO [RS:0;10641b2c0a69:40573 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T10:47:04,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40573-0x10019d731180001, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T10:47:04,390 INFO [RS:0;10641b2c0a69:40573 {}] regionserver.HRegionServer(1031): Exiting; stopping=10641b2c0a69,40573,1730976230476; zookeeper connection closed. 2024-11-07T10:47:04,390 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6e639ec {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6e639ec 2024-11-07T10:47:04,395 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0b29f5d988381aa77506fa3b92944854/.tmp/colfamily11/2149012bcad0457bb1cce674518d4f53 is 52, key is xxx/colfamily11:colfamily11/1730976236850/Put/seqid=0 2024-11-07T10:47:04,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741903_1079 (size=39263) 2024-11-07T10:47:04,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741903_1079 (size=39263) 2024-11-07T10:47:04,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741903_1079 (size=39263) 2024-11-07T10:47:04,403 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0b29f5d988381aa77506fa3b92944854/.tmp/colfamily11/2149012bcad0457bb1cce674518d4f53 2024-11-07T10:47:04,409 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0b29f5d988381aa77506fa3b92944854/.tmp/colfamily11/2149012bcad0457bb1cce674518d4f53 as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0b29f5d988381aa77506fa3b92944854/colfamily11/2149012bcad0457bb1cce674518d4f53 2024-11-07T10:47:04,414 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0b29f5d988381aa77506fa3b92944854/colfamily11/2149012bcad0457bb1cce674518d4f53, entries=703, sequenceid=5, filesize=38.3 K 2024-11-07T10:47:04,415 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 0b29f5d988381aa77506fa3b92944854 in 37ms, sequenceid=5, compaction requested=false 2024-11-07T10:47:04,418 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0b29f5d988381aa77506fa3b92944854/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-07T10:47:04,419 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1730976232769.0b29f5d988381aa77506fa3b92944854. 2024-11-07T10:47:04,419 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0b29f5d988381aa77506fa3b92944854: Waiting for close lock at 1730976424378Running coprocessor pre-close hooks at 1730976424378Disabling compacts and flushes for region at 1730976424378Disabling writes for close at 1730976424378Obtaining lock to block concurrent updates at 1730976424378Preparing flush snapshotting stores in 0b29f5d988381aa77506fa3b92944854 at 1730976424378Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1730976232769.0b29f5d988381aa77506fa3b92944854., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1730976424378Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1730976232769.0b29f5d988381aa77506fa3b92944854. at 1730976424379 (+1 ms)Flushing 0b29f5d988381aa77506fa3b92944854/colfamily11: creating writer at 1730976424379Flushing 0b29f5d988381aa77506fa3b92944854/colfamily11: appending metadata at 1730976424395 (+16 ms)Flushing 0b29f5d988381aa77506fa3b92944854/colfamily11: closing flushed file at 1730976424395Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6886789e: reopening flushed file at 1730976424408 (+13 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 0b29f5d988381aa77506fa3b92944854 in 37ms, sequenceid=5, compaction requested=false at 1730976424415 (+7 ms)Writing region close event to WAL at 1730976424415Running coprocessor post-close hooks at 1730976424419 (+4 ms)Closed at 1730976424419 2024-11-07T10:47:04,419 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1730976232769.0b29f5d988381aa77506fa3b92944854. 2024-11-07T10:47:04,419 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 541874b6b96014b10121ede508f1f61f, disabling compactions & flushes 2024-11-07T10:47:04,419 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f. 2024-11-07T10:47:04,419 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f. 2024-11-07T10:47:04,419 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f. after waiting 0 ms 2024-11-07T10:47:04,419 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f. 2024-11-07T10:47:04,425 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/541874b6b96014b10121ede508f1f61f/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-07T10:47:04,425 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f. 2024-11-07T10:47:04,426 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 541874b6b96014b10121ede508f1f61f: Waiting for close lock at 1730976424419Running coprocessor pre-close hooks at 1730976424419Disabling compacts and flushes for region at 1730976424419Disabling writes for close at 1730976424419Writing region close event to WAL at 1730976424420 (+1 ms)Running coprocessor post-close hooks at 1730976424425 (+5 ms)Closed at 1730976424425 2024-11-07T10:47:04,426 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1730976232769.541874b6b96014b10121ede508f1f61f. 2024-11-07T10:47:04,426 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3f6021bbda5b8b7b00318a57f02cd77d, disabling compactions & flushes 2024-11-07T10:47:04,426 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d. 2024-11-07T10:47:04,426 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d. 2024-11-07T10:47:04,426 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d. after waiting 0 ms 2024-11-07T10:47:04,426 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d. 2024-11-07T10:47:04,426 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 3f6021bbda5b8b7b00318a57f02cd77d 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-07T10:47:04,455 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3f6021bbda5b8b7b00318a57f02cd77d/.tmp/colfamily11/9aa31e0361d64affa46ca1ca755a0ed3 is 52, key is eee/colfamily11:colfamily11/1730976236609/Put/seqid=0 2024-11-07T10:47:04,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741904_1080 (size=39263) 2024-11-07T10:47:04,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741904_1080 (size=39263) 2024-11-07T10:47:04,463 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-07T10:47:04,463 DEBUG [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(1325): Online Regions={3f6021bbda5b8b7b00318a57f02cd77d=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d.} 2024-11-07T10:47:04,463 DEBUG [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(1351): Waiting on 3f6021bbda5b8b7b00318a57f02cd77d 2024-11-07T10:47:04,463 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3f6021bbda5b8b7b00318a57f02cd77d/.tmp/colfamily11/9aa31e0361d64affa46ca1ca755a0ed3 2024-11-07T10:47:04,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741904_1080 (size=39263) 2024-11-07T10:47:04,469 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3f6021bbda5b8b7b00318a57f02cd77d/.tmp/colfamily11/9aa31e0361d64affa46ca1ca755a0ed3 as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3f6021bbda5b8b7b00318a57f02cd77d/colfamily11/9aa31e0361d64affa46ca1ca755a0ed3 2024-11-07T10:47:04,474 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3f6021bbda5b8b7b00318a57f02cd77d/colfamily11/9aa31e0361d64affa46ca1ca755a0ed3, entries=703, sequenceid=5, filesize=38.3 K 2024-11-07T10:47:04,475 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 3f6021bbda5b8b7b00318a57f02cd77d in 49ms, sequenceid=5, compaction requested=false 2024-11-07T10:47:04,478 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3f6021bbda5b8b7b00318a57f02cd77d/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-07T10:47:04,479 INFO [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d. 2024-11-07T10:47:04,479 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3f6021bbda5b8b7b00318a57f02cd77d: Waiting for close lock at 1730976424426Running coprocessor pre-close hooks at 1730976424426Disabling compacts and flushes for region at 1730976424426Disabling writes for close at 1730976424426Obtaining lock to block concurrent updates at 1730976424426Preparing flush snapshotting stores in 3f6021bbda5b8b7b00318a57f02cd77d at 1730976424426Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1730976424426Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d. at 1730976424427 (+1 ms)Flushing 3f6021bbda5b8b7b00318a57f02cd77d/colfamily11: creating writer at 1730976424427Flushing 3f6021bbda5b8b7b00318a57f02cd77d/colfamily11: appending metadata at 1730976424455 (+28 ms)Flushing 3f6021bbda5b8b7b00318a57f02cd77d/colfamily11: closing flushed file at 1730976424455Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19873c68: reopening flushed file at 1730976424468 (+13 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 3f6021bbda5b8b7b00318a57f02cd77d in 49ms, sequenceid=5, compaction requested=false at 1730976424475 (+7 ms)Writing region close event to WAL at 1730976424475Running coprocessor post-close hooks at 1730976424479 (+4 ms)Closed at 1730976424479 2024-11-07T10:47:04,479 DEBUG [RS_CLOSE_REGION-regionserver/10641b2c0a69:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1730976232769.3f6021bbda5b8b7b00318a57f02cd77d. 2024-11-07T10:47:04,663 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(976): stopping server 10641b2c0a69,39013,1730976230620; all regions closed. 2024-11-07T10:47:04,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741833_1009 (size=314313) 2024-11-07T10:47:04,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741833_1009 (size=314313) 2024-11-07T10:47:04,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741833_1009 (size=314313) 2024-11-07T10:47:04,668 DEBUG [RS:2;10641b2c0a69:39013 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/oldWALs 2024-11-07T10:47:04,668 INFO [RS:2;10641b2c0a69:39013 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 10641b2c0a69%2C39013%2C1730976230620:(num 1730976231927) 2024-11-07T10:47:04,668 DEBUG [RS:2;10641b2c0a69:39013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-07T10:47:04,669 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.LeaseManager(133): Closed leases 2024-11-07T10:47:04,669 INFO [RS:2;10641b2c0a69:39013 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T10:47:04,669 INFO [RS:2;10641b2c0a69:39013 {}] hbase.ChoreService(370): Chore service for: regionserver/10641b2c0a69:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-07T10:47:04,669 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-07T10:47:04,669 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-07T10:47:04,669 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-07T10:47:04,669 INFO [RS:2;10641b2c0a69:39013 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T10:47:04,669 INFO [regionserver/10641b2c0a69:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T10:47:04,669 INFO [RS:2;10641b2c0a69:39013 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39013 2024-11-07T10:47:04,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/10641b2c0a69,39013,1730976230620 2024-11-07T10:47:04,671 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-07T10:47:04,671 INFO [RS:2;10641b2c0a69:39013 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T10:47:04,673 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [10641b2c0a69,39013,1730976230620] 2024-11-07T10:47:04,674 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/10641b2c0a69,39013,1730976230620 already deleted, retry=false 2024-11-07T10:47:04,674 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 10641b2c0a69,39013,1730976230620 expired; onlineServers=0 2024-11-07T10:47:04,675 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '10641b2c0a69,43605,1730976229810' ***** 2024-11-07T10:47:04,675 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-07T10:47:04,675 INFO [M:0;10641b2c0a69:43605 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-07T10:47:04,675 INFO [M:0;10641b2c0a69:43605 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-07T10:47:04,675 DEBUG [M:0;10641b2c0a69:43605 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-07T10:47:04,675 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-07T10:47:04,675 DEBUG [M:0;10641b2c0a69:43605 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-07T10:47:04,675 DEBUG [master/10641b2c0a69:0:becomeActiveMaster-HFileCleaner.small.0-1730976231570 {}] cleaner.HFileCleaner(306): Exit Thread[master/10641b2c0a69:0:becomeActiveMaster-HFileCleaner.small.0-1730976231570,5,FailOnTimeoutGroup] 2024-11-07T10:47:04,675 DEBUG [master/10641b2c0a69:0:becomeActiveMaster-HFileCleaner.large.0-1730976231568 {}] cleaner.HFileCleaner(306): Exit Thread[master/10641b2c0a69:0:becomeActiveMaster-HFileCleaner.large.0-1730976231568,5,FailOnTimeoutGroup] 2024-11-07T10:47:04,675 INFO [M:0;10641b2c0a69:43605 {}] hbase.ChoreService(370): Chore service for: master/10641b2c0a69:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-07T10:47:04,675 INFO [M:0;10641b2c0a69:43605 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-07T10:47:04,675 DEBUG [M:0;10641b2c0a69:43605 {}] master.HMaster(1795): Stopping service threads 2024-11-07T10:47:04,675 INFO [M:0;10641b2c0a69:43605 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-07T10:47:04,676 INFO [M:0;10641b2c0a69:43605 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-07T10:47:04,676 INFO [M:0;10641b2c0a69:43605 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-07T10:47:04,676 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-07T10:47:04,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-07T10:47:04,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-07T10:47:04,677 DEBUG [M:0;10641b2c0a69:43605 {}] zookeeper.ZKUtil(347): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-07T10:47:04,677 WARN [M:0;10641b2c0a69:43605 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-07T10:47:04,678 INFO [M:0;10641b2c0a69:43605 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/.lastflushedseqids 2024-11-07T10:47:04,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741905_1081 (size=2837) 2024-11-07T10:47:04,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741905_1081 (size=2837) 2024-11-07T10:47:04,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741905_1081 (size=2837) 2024-11-07T10:47:04,692 INFO [M:0;10641b2c0a69:43605 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-07T10:47:04,692 INFO [M:0;10641b2c0a69:43605 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-07T10:47:04,692 DEBUG [M:0;10641b2c0a69:43605 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-07T10:47:04,692 INFO [M:0;10641b2c0a69:43605 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T10:47:04,692 DEBUG [M:0;10641b2c0a69:43605 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T10:47:04,692 DEBUG [M:0;10641b2c0a69:43605 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-07T10:47:04,692 DEBUG [M:0;10641b2c0a69:43605 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T10:47:04,692 INFO [M:0;10641b2c0a69:43605 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=344.57 KB heapSize=404.97 KB 2024-11-07T10:47:04,709 DEBUG [M:0;10641b2c0a69:43605 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e3d17800588645978d44faedb211d8bd is 82, key is hbase:meta,,1/info:regioninfo/1730976418492/Put/seqid=0 2024-11-07T10:47:04,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741906_1082 (size=6071) 2024-11-07T10:47:04,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741906_1082 (size=6071) 2024-11-07T10:47:04,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741906_1082 (size=6071) 2024-11-07T10:47:04,721 INFO [M:0;10641b2c0a69:43605 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e3d17800588645978d44faedb211d8bd 2024-11-07T10:47:04,742 DEBUG [M:0;10641b2c0a69:43605 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/60f2d26cb17a4d37bda6747714280901 is 4106, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1730976233967/Put/seqid=0 2024-11-07T10:47:04,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741907_1083 (size=13503) 2024-11-07T10:47:04,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741907_1083 (size=13503) 2024-11-07T10:47:04,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741907_1083 (size=13503) 2024-11-07T10:47:04,753 INFO [M:0;10641b2c0a69:43605 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=343.35 KB at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/60f2d26cb17a4d37bda6747714280901 2024-11-07T10:47:04,772 DEBUG [M:0;10641b2c0a69:43605 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ccd8ceabf7f34e8cb2cf96c542da0268 is 69, key is 10641b2c0a69,39013,1730976230620/rs:state/1730976231692/Put/seqid=0 2024-11-07T10:47:04,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T10:47:04,773 INFO [RS:2;10641b2c0a69:39013 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T10:47:04,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39013-0x10019d731180003, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T10:47:04,773 INFO [RS:2;10641b2c0a69:39013 {}] regionserver.HRegionServer(1031): Exiting; stopping=10641b2c0a69,39013,1730976230620; zookeeper connection closed. 2024-11-07T10:47:04,773 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@653bacae {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@653bacae 2024-11-07T10:47:04,774 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-07T10:47:04,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741908_1084 (size=5442) 2024-11-07T10:47:04,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741908_1084 (size=5442) 2024-11-07T10:47:04,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741908_1084 (size=5442) 2024-11-07T10:47:04,779 INFO [M:0;10641b2c0a69:43605 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ccd8ceabf7f34e8cb2cf96c542da0268 2024-11-07T10:47:04,783 INFO [M:0;10641b2c0a69:43605 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ccd8ceabf7f34e8cb2cf96c542da0268 2024-11-07T10:47:04,784 DEBUG [M:0;10641b2c0a69:43605 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e3d17800588645978d44faedb211d8bd as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e3d17800588645978d44faedb211d8bd 2024-11-07T10:47:04,788 INFO [M:0;10641b2c0a69:43605 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e3d17800588645978d44faedb211d8bd, entries=14, sequenceid=635, filesize=5.9 K 2024-11-07T10:47:04,789 DEBUG [M:0;10641b2c0a69:43605 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/60f2d26cb17a4d37bda6747714280901 as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/60f2d26cb17a4d37bda6747714280901 2024-11-07T10:47:04,793 INFO [M:0;10641b2c0a69:43605 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/60f2d26cb17a4d37bda6747714280901, entries=81, sequenceid=635, filesize=13.2 K 2024-11-07T10:47:04,794 DEBUG [M:0;10641b2c0a69:43605 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ccd8ceabf7f34e8cb2cf96c542da0268 as hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ccd8ceabf7f34e8cb2cf96c542da0268 2024-11-07T10:47:04,798 INFO [M:0;10641b2c0a69:43605 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ccd8ceabf7f34e8cb2cf96c542da0268 2024-11-07T10:47:04,798 INFO [M:0;10641b2c0a69:43605 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39973/user/jenkins/test-data/16d0ab7c-ff3f-2ec6-29b7-58e91adb2393/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ccd8ceabf7f34e8cb2cf96c542da0268, entries=3, sequenceid=635, filesize=5.3 K 2024-11-07T10:47:04,799 INFO [M:0;10641b2c0a69:43605 {}] regionserver.HRegion(3140): Finished flush of dataSize ~344.57 KB/352844, heapSize ~404.67 KB/414384, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 107ms, sequenceid=635, compaction requested=false 2024-11-07T10:47:04,800 INFO [M:0;10641b2c0a69:43605 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-07T10:47:04,800 DEBUG [M:0;10641b2c0a69:43605 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1730976424692Disabling compacts and flushes for region at 1730976424692Disabling writes for close at 1730976424692Obtaining lock to block concurrent updates at 1730976424692Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1730976424692Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=352844, getHeapSize=414624, getOffHeapSize=0, getCellsCount=800 at 1730976424693 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1730976424693Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1730976424693Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1730976424709 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1730976424709Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1730976424726 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1730976424742 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1730976424742Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1730976424758 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1730976424772 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1730976424772Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f8b8307: reopening flushed file at 1730976424783 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@810c6d7: reopening flushed file at 1730976424789 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fab7e04: reopening flushed file at 1730976424793 (+4 ms)Finished flush of dataSize ~344.57 KB/352844, heapSize ~404.67 KB/414384, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 107ms, sequenceid=635, compaction requested=false at 1730976424799 (+6 ms)Writing region close event to WAL at 1730976424800 (+1 ms)Closed at 1730976424800 2024-11-07T10:47:04,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40045 is added to blk_1073741830_1006 (size=395198) 2024-11-07T10:47:04,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41673 is added to blk_1073741830_1006 (size=395198) 2024-11-07T10:47:04,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33357 is added to blk_1073741830_1006 (size=395198) 2024-11-07T10:47:04,803 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-07T10:47:04,803 INFO [M:0;10641b2c0a69:43605 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-07T10:47:04,804 INFO [M:0;10641b2c0a69:43605 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43605 2024-11-07T10:47:04,804 INFO [M:0;10641b2c0a69:43605 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-07T10:47:04,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T10:47:04,906 INFO [M:0;10641b2c0a69:43605 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-07T10:47:04,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43605-0x10019d731180000, quorum=127.0.0.1:59788, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-07T10:47:04,911 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4d4e6c58{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T10:47:04,913 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@57f81b0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-07T10:47:04,913 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-07T10:47:04,913 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@30067af9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-07T10:47:04,914 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7f9dc0ae{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/hadoop.log.dir/,STOPPED} 2024-11-07T10:47:04,916 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-07T10:47:04,916 WARN [BP-1776563886-172.17.0.2-1730976226828 heartbeating to localhost/127.0.0.1:39973 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-07T10:47:04,916 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-07T10:47:04,916 WARN [BP-1776563886-172.17.0.2-1730976226828 heartbeating to localhost/127.0.0.1:39973 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1776563886-172.17.0.2-1730976226828 (Datanode Uuid 7110ba11-93cf-48e7-b579-b2e0d9f18deb) service to localhost/127.0.0.1:39973 2024-11-07T10:47:04,918 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/cluster_1da8066b-c576-62ef-93bb-fd1753eba61c/data/data5/current/BP-1776563886-172.17.0.2-1730976226828 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T10:47:04,918 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/cluster_1da8066b-c576-62ef-93bb-fd1753eba61c/data/data6/current/BP-1776563886-172.17.0.2-1730976226828 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T10:47:04,919 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-07T10:47:04,920 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@43a0d547{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T10:47:04,921 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7b1b9e11{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-07T10:47:04,921 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-07T10:47:04,921 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7f4017e5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-07T10:47:04,921 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7186ce9f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/hadoop.log.dir/,STOPPED} 2024-11-07T10:47:04,922 WARN [BP-1776563886-172.17.0.2-1730976226828 heartbeating to localhost/127.0.0.1:39973 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-07T10:47:04,922 WARN [BP-1776563886-172.17.0.2-1730976226828 heartbeating to localhost/127.0.0.1:39973 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1776563886-172.17.0.2-1730976226828 (Datanode Uuid 07928241-511d-4e49-af54-949e15b4e3e7) service to localhost/127.0.0.1:39973 2024-11-07T10:47:04,922 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-07T10:47:04,922 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-07T10:47:04,923 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/cluster_1da8066b-c576-62ef-93bb-fd1753eba61c/data/data3/current/BP-1776563886-172.17.0.2-1730976226828 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T10:47:04,923 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/cluster_1da8066b-c576-62ef-93bb-fd1753eba61c/data/data4/current/BP-1776563886-172.17.0.2-1730976226828 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T10:47:04,923 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-07T10:47:04,928 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1d8160fd{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-07T10:47:04,929 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7e504529{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-07T10:47:04,929 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-07T10:47:04,929 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7da31712{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-07T10:47:04,929 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93e0721{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/hadoop.log.dir/,STOPPED} 2024-11-07T10:47:04,930 WARN [BP-1776563886-172.17.0.2-1730976226828 heartbeating to localhost/127.0.0.1:39973 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-07T10:47:04,930 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-07T10:47:04,930 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-07T10:47:04,930 WARN [BP-1776563886-172.17.0.2-1730976226828 heartbeating to localhost/127.0.0.1:39973 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1776563886-172.17.0.2-1730976226828 (Datanode Uuid 107098e0-f1c7-4f05-8e1d-4b90535e9624) service to localhost/127.0.0.1:39973 2024-11-07T10:47:04,931 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/cluster_1da8066b-c576-62ef-93bb-fd1753eba61c/data/data1/current/BP-1776563886-172.17.0.2-1730976226828 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T10:47:04,931 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/cluster_1da8066b-c576-62ef-93bb-fd1753eba61c/data/data2/current/BP-1776563886-172.17.0.2-1730976226828 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-07T10:47:04,931 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-07T10:47:04,942 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5183f126{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-07T10:47:04,942 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4350931a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-07T10:47:04,942 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-07T10:47:04,942 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4974766f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-07T10:47:04,943 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2c447bb0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c0d88c3c-3b9d-9dab-4725-48467ef5cc51/hadoop.log.dir/,STOPPED} 2024-11-07T10:47:04,958 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-07T10:47:05,034 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-07T10:47:05,045 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.procedure.TestSCPWithMeta#testRecoveryAndDoubleExecutionOnRsWithMeta Thread=90 (was 11) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:39973 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: BootstrapNodeManager java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_META-regionserver/10641b2c0a69:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/10641b2c0a69:0.Chore.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39973 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:39973 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: nioEventLoopGroup-3-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (127692218) connection to localhost/127.0.0.1:39973 from jenkins.hfs.0 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/10641b2c0a69:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/10641b2c0a69:0.logRoller java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:180) Potentially hanging thread: nioEventLoopGroup-6-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SSL Certificates Store Monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/10641b2c0a69:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: nioEventLoopGroup-2-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:39973 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/10641b2c0a69:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39973 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-7-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-6-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: nioEventLoopGroup-7-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (127692218) connection to localhost/127.0.0.1:39973 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-5-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Abort regionserver monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Client (127692218) connection to localhost/127.0.0.1:39973 from jenkins.hfs.2 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: ForkJoinPool-2-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39973 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (127692218) connection to localhost/127.0.0.1:39973 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-6-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@53c35b41 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-7-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/10641b2c0a69:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39973 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/10641b2c0a69:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/10641b2c0a69:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=452 (was 285) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=160 (was 330), ProcessCount=11 (was 11), AvailableMemoryMB=4812 (was 5114)