2024-11-16 05:51:19,902 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-16 05:51:19,920 main DEBUG Took 0.015982 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-16 05:51:19,921 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-16 05:51:19,921 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-16 05:51:19,922 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-16 05:51:19,923 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:51:19,930 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-16 05:51:19,943 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:51:19,945 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:51:19,946 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:51:19,947 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:51:19,948 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:51:19,948 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:51:19,951 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:51:19,951 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:51:19,952 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:51:19,952 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:51:19,954 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:51:19,954 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:51:19,955 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:51:19,956 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:51:19,956 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:51:19,957 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:51:19,958 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:51:19,958 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:51:19,959 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:51:19,960 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:51:19,960 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:51:19,961 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:51:19,962 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:51:19,962 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 05:51:19,963 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:51:19,964 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-16 05:51:19,966 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 05:51:19,967 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-16 05:51:19,970 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-16 05:51:19,971 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-16 05:51:19,973 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-16 05:51:19,974 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-16 05:51:19,984 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-16 05:51:19,988 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-16 05:51:19,991 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-16 05:51:19,992 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-16 05:51:19,992 main DEBUG createAppenders(={Console}) 2024-11-16 05:51:19,993 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-16 05:51:19,994 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-16 05:51:19,994 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-16 05:51:19,995 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-16 05:51:19,995 main DEBUG OutputStream closed 2024-11-16 05:51:19,995 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-16 05:51:19,995 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-16 05:51:19,996 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-16 05:51:20,094 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-16 05:51:20,097 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-16 05:51:20,099 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-16 05:51:20,100 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-16 05:51:20,100 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-16 05:51:20,101 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-16 05:51:20,101 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-16 05:51:20,102 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-16 05:51:20,102 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-16 05:51:20,102 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-16 05:51:20,103 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-16 05:51:20,103 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-16 05:51:20,103 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-16 05:51:20,104 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-16 05:51:20,104 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-16 05:51:20,104 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-16 05:51:20,105 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-16 05:51:20,106 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-16 05:51:20,108 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16 05:51:20,109 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-16 05:51:20,109 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-16 05:51:20,110 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-16T05:51:20,122 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta timeout: 13 mins 2024-11-16 05:51:20,124 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-16 05:51:20,125 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16T05:51:20,158 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.procedure.TestSCPWithMeta#testRecoveryAndDoubleExecutionOnRsWithMeta Thread=11, OpenFileDescriptor=285, MaxFileDescriptor=1048576, SystemLoadAverage=200, ProcessCount=11, AvailableMemoryMB=4730 2024-11-16T05:51:20,380 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d 2024-11-16T05:51:20,381 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-16T05:51:20,396 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/cluster_59495369-691a-be10-1b61-6f414eddee7f, deleteOnExit=true 2024-11-16T05:51:20,397 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-16T05:51:20,398 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/test.cache.data in system properties and HBase conf 2024-11-16T05:51:20,398 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/hadoop.tmp.dir in system properties and HBase conf 2024-11-16T05:51:20,399 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/hadoop.log.dir in system properties and HBase conf 2024-11-16T05:51:20,400 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-16T05:51:20,400 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-16T05:51:20,401 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-16T05:51:20,482 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-16T05:51:20,591 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T05:51:20,595 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-16T05:51:20,596 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-16T05:51:20,597 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-16T05:51:20,597 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T05:51:20,598 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-16T05:51:20,599 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-16T05:51:20,599 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T05:51:20,600 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T05:51:20,601 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-16T05:51:20,602 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/nfs.dump.dir in system properties and HBase conf 2024-11-16T05:51:20,603 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/java.io.tmpdir in system properties and HBase conf 2024-11-16T05:51:20,603 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T05:51:20,604 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-16T05:51:20,605 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-16T05:51:21,527 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-16T05:51:21,621 INFO [Time-limited test {}] log.Log(170): Logging initialized @2424ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-16T05:51:21,716 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T05:51:21,796 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T05:51:21,826 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T05:51:21,827 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T05:51:21,828 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-16T05:51:21,845 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T05:51:21,850 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2c447bb0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/hadoop.log.dir/,AVAILABLE} 2024-11-16T05:51:21,852 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4974766f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T05:51:22,092 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5183f126{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/java.io.tmpdir/jetty-localhost-42259-hadoop-hdfs-3_4_1-tests_jar-_-any-15896003971757989072/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T05:51:22,102 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4350931a{HTTP/1.1, (http/1.1)}{localhost:42259} 2024-11-16T05:51:22,102 INFO [Time-limited test {}] server.Server(415): Started @2907ms 2024-11-16T05:51:22,554 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T05:51:22,563 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T05:51:22,566 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T05:51:22,567 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T05:51:22,567 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-16T05:51:22,569 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7da31712{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/hadoop.log.dir/,AVAILABLE} 2024-11-16T05:51:22,570 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@664a7783{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T05:51:22,676 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4d81bf0e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/java.io.tmpdir/jetty-localhost-34163-hadoop-hdfs-3_4_1-tests_jar-_-any-7328206374100498545/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T05:51:22,677 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@572fe942{HTTP/1.1, (http/1.1)}{localhost:34163} 2024-11-16T05:51:22,677 INFO [Time-limited test {}] server.Server(415): Started @3481ms 2024-11-16T05:51:22,734 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-16T05:51:22,893 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T05:51:22,908 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T05:51:22,925 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T05:51:22,925 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T05:51:22,926 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-16T05:51:22,929 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7f4017e5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/hadoop.log.dir/,AVAILABLE} 2024-11-16T05:51:22,930 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6c516a28{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T05:51:23,069 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1ba455be{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/java.io.tmpdir/jetty-localhost-41883-hadoop-hdfs-3_4_1-tests_jar-_-any-3208987200536975954/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T05:51:23,071 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@259771a6{HTTP/1.1, (http/1.1)}{localhost:41883} 2024-11-16T05:51:23,071 INFO [Time-limited test {}] server.Server(415): Started @3875ms 2024-11-16T05:51:23,075 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-16T05:51:23,168 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T05:51:23,174 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T05:51:23,194 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T05:51:23,194 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T05:51:23,195 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-16T05:51:23,196 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@30067af9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/hadoop.log.dir/,AVAILABLE} 2024-11-16T05:51:23,196 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@17888407{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T05:51:23,267 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/cluster_59495369-691a-be10-1b61-6f414eddee7f/data/data1/current/BP-1182044883-172.17.0.2-1731736281266/current, will proceed with Du for space computation calculation, 2024-11-16T05:51:23,267 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/cluster_59495369-691a-be10-1b61-6f414eddee7f/data/data3/current/BP-1182044883-172.17.0.2-1731736281266/current, will proceed with Du for space computation calculation, 2024-11-16T05:51:23,270 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/cluster_59495369-691a-be10-1b61-6f414eddee7f/data/data4/current/BP-1182044883-172.17.0.2-1731736281266/current, will proceed with Du for space computation calculation, 2024-11-16T05:51:23,271 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/cluster_59495369-691a-be10-1b61-6f414eddee7f/data/data2/current/BP-1182044883-172.17.0.2-1731736281266/current, will proceed with Du for space computation calculation, 2024-11-16T05:51:23,337 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-16T05:51:23,341 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-16T05:51:23,343 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@22333f25{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/java.io.tmpdir/jetty-localhost-43437-hadoop-hdfs-3_4_1-tests_jar-_-any-12687214788199642033/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T05:51:23,344 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@eec57c1{HTTP/1.1, (http/1.1)}{localhost:43437} 2024-11-16T05:51:23,344 INFO [Time-limited test {}] server.Server(415): Started @4149ms 2024-11-16T05:51:23,348 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-16T05:51:23,431 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x27b275a0ef452008 with lease ID 0x2a813d4d024c7747: Processing first storage report for DS-f0a76934-b7f3-4998-a613-d00266c6901a from datanode DatanodeRegistration(127.0.0.1:33441, datanodeUuid=c5d2e700-f00c-4f38-96af-fd6fb0055b8a, infoPort=37155, infoSecurePort=0, ipcPort=40719, storageInfo=lv=-57;cid=testClusterID;nsid=2099212532;c=1731736281266) 2024-11-16T05:51:23,433 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x27b275a0ef452008 with lease ID 0x2a813d4d024c7747: from storage DS-f0a76934-b7f3-4998-a613-d00266c6901a node DatanodeRegistration(127.0.0.1:33441, datanodeUuid=c5d2e700-f00c-4f38-96af-fd6fb0055b8a, infoPort=37155, infoSecurePort=0, ipcPort=40719, storageInfo=lv=-57;cid=testClusterID;nsid=2099212532;c=1731736281266), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-16T05:51:23,433 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x32fab283ba093aa6 with lease ID 0x2a813d4d024c7746: Processing first storage report for DS-56cc9d96-a128-48c4-add0-03db81c0fd8d from datanode DatanodeRegistration(127.0.0.1:32873, datanodeUuid=c8992440-ff96-4231-ad71-71d7f7ce1556, infoPort=43727, infoSecurePort=0, ipcPort=40157, storageInfo=lv=-57;cid=testClusterID;nsid=2099212532;c=1731736281266) 2024-11-16T05:51:23,434 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x32fab283ba093aa6 with lease ID 0x2a813d4d024c7746: from storage DS-56cc9d96-a128-48c4-add0-03db81c0fd8d node DatanodeRegistration(127.0.0.1:32873, datanodeUuid=c8992440-ff96-4231-ad71-71d7f7ce1556, infoPort=43727, infoSecurePort=0, ipcPort=40157, storageInfo=lv=-57;cid=testClusterID;nsid=2099212532;c=1731736281266), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T05:51:23,434 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x27b275a0ef452008 with lease ID 0x2a813d4d024c7747: Processing first storage report for DS-e00a4a53-6360-4333-b70f-0173aff0ed5e from datanode DatanodeRegistration(127.0.0.1:33441, datanodeUuid=c5d2e700-f00c-4f38-96af-fd6fb0055b8a, infoPort=37155, infoSecurePort=0, ipcPort=40719, storageInfo=lv=-57;cid=testClusterID;nsid=2099212532;c=1731736281266) 2024-11-16T05:51:23,434 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x27b275a0ef452008 with lease ID 0x2a813d4d024c7747: from storage DS-e00a4a53-6360-4333-b70f-0173aff0ed5e node DatanodeRegistration(127.0.0.1:33441, datanodeUuid=c5d2e700-f00c-4f38-96af-fd6fb0055b8a, infoPort=37155, infoSecurePort=0, ipcPort=40719, storageInfo=lv=-57;cid=testClusterID;nsid=2099212532;c=1731736281266), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T05:51:23,434 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x32fab283ba093aa6 with lease ID 0x2a813d4d024c7746: Processing first storage report for DS-1105c40d-9127-4180-b321-0bd43a13f862 from datanode DatanodeRegistration(127.0.0.1:32873, datanodeUuid=c8992440-ff96-4231-ad71-71d7f7ce1556, infoPort=43727, infoSecurePort=0, ipcPort=40157, storageInfo=lv=-57;cid=testClusterID;nsid=2099212532;c=1731736281266) 2024-11-16T05:51:23,435 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x32fab283ba093aa6 with lease ID 0x2a813d4d024c7746: from storage DS-1105c40d-9127-4180-b321-0bd43a13f862 node DatanodeRegistration(127.0.0.1:32873, datanodeUuid=c8992440-ff96-4231-ad71-71d7f7ce1556, infoPort=43727, infoSecurePort=0, ipcPort=40157, storageInfo=lv=-57;cid=testClusterID;nsid=2099212532;c=1731736281266), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T05:51:23,481 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/cluster_59495369-691a-be10-1b61-6f414eddee7f/data/data5/current/BP-1182044883-172.17.0.2-1731736281266/current, will proceed with Du for space computation calculation, 2024-11-16T05:51:23,482 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/cluster_59495369-691a-be10-1b61-6f414eddee7f/data/data6/current/BP-1182044883-172.17.0.2-1731736281266/current, will proceed with Du for space computation calculation, 2024-11-16T05:51:23,521 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-16T05:51:23,531 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x763ac8631ff188a2 with lease ID 0x2a813d4d024c7748: Processing first storage report for DS-d9cb5fed-ca91-420f-8b4c-9e853c8c3426 from datanode DatanodeRegistration(127.0.0.1:36993, datanodeUuid=63e24c74-3be6-46bf-ad1a-7f46f1bfb235, infoPort=37513, infoSecurePort=0, ipcPort=43713, storageInfo=lv=-57;cid=testClusterID;nsid=2099212532;c=1731736281266) 2024-11-16T05:51:23,531 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x763ac8631ff188a2 with lease ID 0x2a813d4d024c7748: from storage DS-d9cb5fed-ca91-420f-8b4c-9e853c8c3426 node DatanodeRegistration(127.0.0.1:36993, datanodeUuid=63e24c74-3be6-46bf-ad1a-7f46f1bfb235, infoPort=37513, infoSecurePort=0, ipcPort=43713, storageInfo=lv=-57;cid=testClusterID;nsid=2099212532;c=1731736281266), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-16T05:51:23,531 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x763ac8631ff188a2 with lease ID 0x2a813d4d024c7748: Processing first storage report for DS-31cd4936-3157-4fca-8c13-c3fa6f4bfd34 from datanode DatanodeRegistration(127.0.0.1:36993, datanodeUuid=63e24c74-3be6-46bf-ad1a-7f46f1bfb235, infoPort=37513, infoSecurePort=0, ipcPort=43713, storageInfo=lv=-57;cid=testClusterID;nsid=2099212532;c=1731736281266) 2024-11-16T05:51:23,532 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x763ac8631ff188a2 with lease ID 0x2a813d4d024c7748: from storage DS-31cd4936-3157-4fca-8c13-c3fa6f4bfd34 node DatanodeRegistration(127.0.0.1:36993, datanodeUuid=63e24c74-3be6-46bf-ad1a-7f46f1bfb235, infoPort=37513, infoSecurePort=0, ipcPort=43713, storageInfo=lv=-57;cid=testClusterID;nsid=2099212532;c=1731736281266), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T05:51:23,731 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d 2024-11-16T05:51:23,821 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/cluster_59495369-691a-be10-1b61-6f414eddee7f/zookeeper_0, clientPort=58523, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/cluster_59495369-691a-be10-1b61-6f414eddee7f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/cluster_59495369-691a-be10-1b61-6f414eddee7f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-16T05:51:23,837 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58523 2024-11-16T05:51:23,852 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:51:23,856 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:51:24,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741825_1001 (size=7) 2024-11-16T05:51:24,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741825_1001 (size=7) 2024-11-16T05:51:24,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741825_1001 (size=7) 2024-11-16T05:51:24,516 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 with version=8 2024-11-16T05:51:24,516 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/hbase-staging 2024-11-16T05:51:24,602 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-16T05:51:24,836 INFO [Time-limited test {}] client.ConnectionUtils(128): master/3456ee6a3164:0 server-side Connection retries=45 2024-11-16T05:51:24,847 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:51:24,848 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T05:51:24,852 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T05:51:24,853 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:51:24,853 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T05:51:25,005 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T05:51:25,068 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-16T05:51:25,078 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-16T05:51:25,081 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T05:51:25,115 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 82911 (auto-detected) 2024-11-16T05:51:25,117 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-16T05:51:25,147 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44685 2024-11-16T05:51:25,174 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44685 connecting to ZooKeeper ensemble=127.0.0.1:58523 2024-11-16T05:51:25,218 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:446850x0, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T05:51:25,223 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44685-0x100471607020000 connected 2024-11-16T05:51:25,281 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:51:25,286 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:51:25,300 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:51:25,305 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8, hbase.cluster.distributed=false 2024-11-16T05:51:25,341 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T05:51:25,353 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44685 2024-11-16T05:51:25,357 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44685 2024-11-16T05:51:25,358 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44685 2024-11-16T05:51:25,365 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44685 2024-11-16T05:51:25,369 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44685 2024-11-16T05:51:25,514 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3456ee6a3164:0 server-side Connection retries=45 2024-11-16T05:51:25,517 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:51:25,518 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T05:51:25,518 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T05:51:25,519 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:51:25,519 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T05:51:25,523 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T05:51:25,529 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T05:51:25,534 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33007 2024-11-16T05:51:25,537 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33007 connecting to ZooKeeper ensemble=127.0.0.1:58523 2024-11-16T05:51:25,538 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:51:25,543 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:51:25,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:330070x0, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T05:51:25,556 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:51:25,556 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33007-0x100471607020001 connected 2024-11-16T05:51:25,561 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T05:51:25,580 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T05:51:25,583 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T05:51:25,593 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T05:51:25,596 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33007 2024-11-16T05:51:25,597 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33007 2024-11-16T05:51:25,601 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33007 2024-11-16T05:51:25,604 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33007 2024-11-16T05:51:25,609 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33007 2024-11-16T05:51:25,633 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3456ee6a3164:0 server-side Connection retries=45 2024-11-16T05:51:25,633 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:51:25,634 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T05:51:25,634 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T05:51:25,634 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:51:25,634 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T05:51:25,635 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T05:51:25,636 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T05:51:25,637 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44781 2024-11-16T05:51:25,639 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44781 connecting to ZooKeeper ensemble=127.0.0.1:58523 2024-11-16T05:51:25,641 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:51:25,645 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:51:25,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:447810x0, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T05:51:25,657 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44781-0x100471607020002 connected 2024-11-16T05:51:25,660 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44781-0x100471607020002, quorum=127.0.0.1:58523, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:51:25,661 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T05:51:25,665 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T05:51:25,667 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44781-0x100471607020002, quorum=127.0.0.1:58523, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T05:51:25,669 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44781-0x100471607020002, quorum=127.0.0.1:58523, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T05:51:25,674 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44781 2024-11-16T05:51:25,675 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44781 2024-11-16T05:51:25,677 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44781 2024-11-16T05:51:25,680 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44781 2024-11-16T05:51:25,681 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44781 2024-11-16T05:51:25,705 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3456ee6a3164:0 server-side Connection retries=45 2024-11-16T05:51:25,706 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:51:25,706 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T05:51:25,706 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T05:51:25,706 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T05:51:25,706 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T05:51:25,706 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T05:51:25,707 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T05:51:25,708 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43415 2024-11-16T05:51:25,711 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43415 connecting to ZooKeeper ensemble=127.0.0.1:58523 2024-11-16T05:51:25,712 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:51:25,716 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:51:25,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:434150x0, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T05:51:25,725 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43415-0x100471607020003 connected 2024-11-16T05:51:25,728 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:51:25,729 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T05:51:25,734 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T05:51:25,735 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T05:51:25,738 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T05:51:25,743 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43415 2024-11-16T05:51:25,745 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43415 2024-11-16T05:51:25,749 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43415 2024-11-16T05:51:25,753 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43415 2024-11-16T05:51:25,754 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43415 2024-11-16T05:51:25,775 DEBUG [M:0;3456ee6a3164:44685 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;3456ee6a3164:44685 2024-11-16T05:51:25,777 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/3456ee6a3164,44685,1731736284655 2024-11-16T05:51:25,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:51:25,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44781-0x100471607020002, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:51:25,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:51:25,785 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:51:25,788 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/3456ee6a3164,44685,1731736284655 2024-11-16T05:51:25,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44781-0x100471607020002, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T05:51:25,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:51:25,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44781-0x100471607020002, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:51:25,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T05:51:25,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T05:51:25,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:51:25,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:51:25,817 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T05:51:25,818 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/3456ee6a3164,44685,1731736284655 from backup master directory 2024-11-16T05:51:25,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/3456ee6a3164,44685,1731736284655 2024-11-16T05:51:25,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44781-0x100471607020002, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:51:25,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:51:25,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:51:25,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T05:51:25,824 WARN [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T05:51:25,825 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=3456ee6a3164,44685,1731736284655 2024-11-16T05:51:25,827 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-16T05:51:25,829 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-16T05:51:25,900 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/hbase.id] with ID: 64bcd7a0-b620-4e04-85b3-a19d6690a50b 2024-11-16T05:51:25,900 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/.tmp/hbase.id 2024-11-16T05:51:25,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741826_1002 (size=42) 2024-11-16T05:51:25,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741826_1002 (size=42) 2024-11-16T05:51:25,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741826_1002 (size=42) 2024-11-16T05:51:25,920 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/.tmp/hbase.id]:[hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/hbase.id] 2024-11-16T05:51:25,984 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T05:51:25,990 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T05:51:26,013 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-11-16T05:51:26,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44781-0x100471607020002, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:51:26,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:51:26,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:51:26,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:51:26,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741827_1003 (size=196) 2024-11-16T05:51:26,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741827_1003 (size=196) 2024-11-16T05:51:26,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741827_1003 (size=196) 2024-11-16T05:51:26,084 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T05:51:26,087 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T05:51:26,113 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:51:26,119 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T05:51:26,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741828_1004 (size=1189) 2024-11-16T05:51:26,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741828_1004 (size=1189) 2024-11-16T05:51:26,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741828_1004 (size=1189) 2024-11-16T05:51:26,202 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/MasterData/data/master/store 2024-11-16T05:51:26,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741829_1005 (size=34) 2024-11-16T05:51:26,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741829_1005 (size=34) 2024-11-16T05:51:26,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741829_1005 (size=34) 2024-11-16T05:51:26,243 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-16T05:51:26,247 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:26,249 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T05:51:26,249 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:51:26,249 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:51:26,250 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T05:51:26,251 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:51:26,251 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:51:26,252 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731736286248Disabling compacts and flushes for region at 1731736286248Disabling writes for close at 1731736286250 (+2 ms)Writing region close event to WAL at 1731736286251 (+1 ms)Closed at 1731736286251 2024-11-16T05:51:26,254 WARN [master/3456ee6a3164:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/MasterData/data/master/store/.initializing 2024-11-16T05:51:26,255 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/MasterData/WALs/3456ee6a3164,44685,1731736284655 2024-11-16T05:51:26,265 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T05:51:26,288 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3456ee6a3164%2C44685%2C1731736284655, suffix=, logDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/MasterData/WALs/3456ee6a3164,44685,1731736284655, archiveDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/MasterData/oldWALs, maxLogs=10 2024-11-16T05:51:26,313 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/MasterData/WALs/3456ee6a3164,44685,1731736284655/3456ee6a3164%2C44685%2C1731736284655.1731736286293, exclude list is [], retry=0 2024-11-16T05:51:26,332 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32873,DS-56cc9d96-a128-48c4-add0-03db81c0fd8d,DISK] 2024-11-16T05:51:26,332 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36993,DS-d9cb5fed-ca91-420f-8b4c-9e853c8c3426,DISK] 2024-11-16T05:51:26,332 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33441,DS-f0a76934-b7f3-4998-a613-d00266c6901a,DISK] 2024-11-16T05:51:26,335 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-16T05:51:26,372 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/MasterData/WALs/3456ee6a3164,44685,1731736284655/3456ee6a3164%2C44685%2C1731736284655.1731736286293 2024-11-16T05:51:26,372 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43727:43727),(127.0.0.1/127.0.0.1:37155:37155),(127.0.0.1/127.0.0.1:37513:37513)] 2024-11-16T05:51:26,373 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:51:26,373 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:26,377 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:51:26,378 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:51:26,412 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:51:26,435 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-16T05:51:26,438 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:26,441 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:51:26,441 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:51:26,444 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-16T05:51:26,445 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:26,446 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:26,447 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:51:26,450 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-16T05:51:26,450 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:26,451 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:26,452 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:51:26,455 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-16T05:51:26,455 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:26,456 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:26,457 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:51:26,460 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:51:26,462 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:51:26,467 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:51:26,468 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:51:26,471 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T05:51:26,475 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T05:51:26,479 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:51:26,480 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63052710, jitterRate=-0.060441404581069946}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T05:51:26,490 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731736286390Initializing all the Stores at 1731736286392 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736286392Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736286393 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736286393Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736286393Cleaning up temporary data from old regions at 1731736286468 (+75 ms)Region opened successfully at 1731736286490 (+22 ms) 2024-11-16T05:51:26,491 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-16T05:51:26,525 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d2ae78c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3456ee6a3164/172.17.0.2:0 2024-11-16T05:51:26,551 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-16T05:51:26,561 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:51:26,561 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:51:26,563 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:51:26,565 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-16T05:51:26,569 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-16T05:51:26,569 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-16T05:51:26,592 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-16T05:51:26,600 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-16T05:51:26,604 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-16T05:51:26,606 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-16T05:51:26,607 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-16T05:51:26,609 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-16T05:51:26,611 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-16T05:51:26,614 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-16T05:51:26,617 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-16T05:51:26,618 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-16T05:51:26,621 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-16T05:51:26,639 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-16T05:51:26,641 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-16T05:51:26,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44781-0x100471607020002, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T05:51:26,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T05:51:26,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T05:51:26,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T05:51:26,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:51:26,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:51:26,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:51:26,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44781-0x100471607020002, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:51:26,650 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=3456ee6a3164,44685,1731736284655, sessionid=0x100471607020000, setting cluster-up flag (Was=false) 2024-11-16T05:51:26,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44781-0x100471607020002, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:51:26,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:51:26,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:51:26,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:51:26,668 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-16T05:51:26,671 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3456ee6a3164,44685,1731736284655 2024-11-16T05:51:26,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44781-0x100471607020002, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:51:26,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:51:26,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:51:26,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:51:26,685 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-16T05:51:26,687 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3456ee6a3164,44685,1731736284655 2024-11-16T05:51:26,693 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-16T05:51:26,758 INFO [RS:1;3456ee6a3164:44781 {}] regionserver.HRegionServer(746): ClusterId : 64bcd7a0-b620-4e04-85b3-a19d6690a50b 2024-11-16T05:51:26,760 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer(746): ClusterId : 64bcd7a0-b620-4e04-85b3-a19d6690a50b 2024-11-16T05:51:26,760 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(746): ClusterId : 64bcd7a0-b620-4e04-85b3-a19d6690a50b 2024-11-16T05:51:26,761 DEBUG [RS:0;3456ee6a3164:33007 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T05:51:26,761 DEBUG [RS:1;3456ee6a3164:44781 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T05:51:26,761 DEBUG [RS:2;3456ee6a3164:43415 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T05:51:26,767 DEBUG [RS:1;3456ee6a3164:44781 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T05:51:26,767 DEBUG [RS:0;3456ee6a3164:33007 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T05:51:26,767 DEBUG [RS:1;3456ee6a3164:44781 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T05:51:26,767 DEBUG [RS:0;3456ee6a3164:33007 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T05:51:26,769 DEBUG [RS:2;3456ee6a3164:43415 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T05:51:26,769 DEBUG [RS:2;3456ee6a3164:43415 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T05:51:26,771 DEBUG [RS:0;3456ee6a3164:33007 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T05:51:26,771 DEBUG [RS:1;3456ee6a3164:44781 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T05:51:26,771 DEBUG [RS:2;3456ee6a3164:43415 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T05:51:26,771 DEBUG [RS:0;3456ee6a3164:33007 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e06d417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3456ee6a3164/172.17.0.2:0 2024-11-16T05:51:26,771 DEBUG [RS:1;3456ee6a3164:44781 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b7ea8a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3456ee6a3164/172.17.0.2:0 2024-11-16T05:51:26,772 DEBUG [RS:2;3456ee6a3164:43415 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c28c2a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3456ee6a3164/172.17.0.2:0 2024-11-16T05:51:26,775 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:51:26,786 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-16T05:51:26,794 DEBUG [RS:2;3456ee6a3164:43415 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;3456ee6a3164:43415 2024-11-16T05:51:26,795 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-16T05:51:26,796 DEBUG [RS:0;3456ee6a3164:33007 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;3456ee6a3164:33007 2024-11-16T05:51:26,796 DEBUG [RS:1;3456ee6a3164:44781 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;3456ee6a3164:44781 2024-11-16T05:51:26,798 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T05:51:26,798 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T05:51:26,798 INFO [RS:1;3456ee6a3164:44781 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T05:51:26,798 INFO [RS:1;3456ee6a3164:44781 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T05:51:26,798 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T05:51:26,798 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T05:51:26,798 DEBUG [RS:1;3456ee6a3164:44781 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T05:51:26,798 DEBUG [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T05:51:26,798 DEBUG [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T05:51:26,801 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer(2659): reportForDuty to master=3456ee6a3164,44685,1731736284655 with port=33007, startcode=1731736285465 2024-11-16T05:51:26,801 INFO [RS:1;3456ee6a3164:44781 {}] regionserver.HRegionServer(2659): reportForDuty to master=3456ee6a3164,44685,1731736284655 with port=44781, startcode=1731736285630 2024-11-16T05:51:26,801 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(2659): reportForDuty to master=3456ee6a3164,44685,1731736284655 with port=43415, startcode=1731736285705 2024-11-16T05:51:26,802 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 3456ee6a3164,44685,1731736284655 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-16T05:51:26,810 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/3456ee6a3164:0, corePoolSize=5, maxPoolSize=5 2024-11-16T05:51:26,810 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/3456ee6a3164:0, corePoolSize=5, maxPoolSize=5 2024-11-16T05:51:26,811 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/3456ee6a3164:0, corePoolSize=5, maxPoolSize=5 2024-11-16T05:51:26,811 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/3456ee6a3164:0, corePoolSize=5, maxPoolSize=5 2024-11-16T05:51:26,811 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/3456ee6a3164:0, corePoolSize=10, maxPoolSize=10 2024-11-16T05:51:26,811 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:26,812 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/3456ee6a3164:0, corePoolSize=2, maxPoolSize=2 2024-11-16T05:51:26,812 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:26,813 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736316813 2024-11-16T05:51:26,815 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-16T05:51:26,815 DEBUG [RS:1;3456ee6a3164:44781 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T05:51:26,815 DEBUG [RS:0;3456ee6a3164:33007 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T05:51:26,815 DEBUG [RS:2;3456ee6a3164:43415 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T05:51:26,817 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-16T05:51:26,819 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T05:51:26,820 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-16T05:51:26,820 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-16T05:51:26,821 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-16T05:51:26,821 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-16T05:51:26,821 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-16T05:51:26,826 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:26,825 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:26,826 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-16T05:51:26,829 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-16T05:51:26,832 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-16T05:51:26,833 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-16T05:51:26,837 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-16T05:51:26,838 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-16T05:51:26,852 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.large.0-1731736286842,5,FailOnTimeoutGroup] 2024-11-16T05:51:26,854 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.small.0-1731736286853,5,FailOnTimeoutGroup] 2024-11-16T05:51:26,854 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:26,854 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-16T05:51:26,856 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:26,856 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:26,868 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35203, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T05:51:26,868 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38469, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T05:51:26,868 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57829, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T05:51:26,907 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44685 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3456ee6a3164,43415,1731736285705 2024-11-16T05:51:26,909 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44685 {}] master.ServerManager(517): Registering regionserver=3456ee6a3164,43415,1731736285705 2024-11-16T05:51:26,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741831_1007 (size=1321) 2024-11-16T05:51:26,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741831_1007 (size=1321) 2024-11-16T05:51:26,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741831_1007 (size=1321) 2024-11-16T05:51:26,921 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44685 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3456ee6a3164,33007,1731736285465 2024-11-16T05:51:26,921 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44685 {}] master.ServerManager(517): Registering regionserver=3456ee6a3164,33007,1731736285465 2024-11-16T05:51:26,922 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-16T05:51:26,923 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 2024-11-16T05:51:26,926 DEBUG [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 2024-11-16T05:51:26,926 DEBUG [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45275 2024-11-16T05:51:26,926 DEBUG [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T05:51:26,928 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44685 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3456ee6a3164,44781,1731736285630 2024-11-16T05:51:26,928 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44685 {}] master.ServerManager(517): Registering regionserver=3456ee6a3164,44781,1731736285630 2024-11-16T05:51:26,929 DEBUG [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 2024-11-16T05:51:26,929 DEBUG [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45275 2024-11-16T05:51:26,929 DEBUG [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T05:51:26,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T05:51:26,932 DEBUG [RS:1;3456ee6a3164:44781 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 2024-11-16T05:51:26,933 DEBUG [RS:1;3456ee6a3164:44781 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45275 2024-11-16T05:51:26,933 DEBUG [RS:1;3456ee6a3164:44781 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T05:51:26,936 DEBUG [RS:2;3456ee6a3164:43415 {}] zookeeper.ZKUtil(111): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3456ee6a3164,43415,1731736285705 2024-11-16T05:51:26,936 WARN [RS:2;3456ee6a3164:43415 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T05:51:26,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T05:51:26,937 INFO [RS:2;3456ee6a3164:43415 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T05:51:26,937 DEBUG [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,43415,1731736285705 2024-11-16T05:51:26,937 DEBUG [RS:0;3456ee6a3164:33007 {}] zookeeper.ZKUtil(111): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3456ee6a3164,33007,1731736285465 2024-11-16T05:51:26,937 DEBUG [RS:1;3456ee6a3164:44781 {}] zookeeper.ZKUtil(111): regionserver:44781-0x100471607020002, quorum=127.0.0.1:58523, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3456ee6a3164,44781,1731736285630 2024-11-16T05:51:26,937 WARN [RS:0;3456ee6a3164:33007 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T05:51:26,937 WARN [RS:1;3456ee6a3164:44781 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T05:51:26,937 INFO [RS:0;3456ee6a3164:33007 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T05:51:26,938 INFO [RS:1;3456ee6a3164:44781 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T05:51:26,938 DEBUG [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,33007,1731736285465 2024-11-16T05:51:26,938 DEBUG [RS:1;3456ee6a3164:44781 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630 2024-11-16T05:51:26,943 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3456ee6a3164,33007,1731736285465] 2024-11-16T05:51:26,943 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3456ee6a3164,43415,1731736285705] 2024-11-16T05:51:26,944 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3456ee6a3164,44781,1731736285630] 2024-11-16T05:51:26,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741832_1008 (size=32) 2024-11-16T05:51:26,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741832_1008 (size=32) 2024-11-16T05:51:26,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741832_1008 (size=32) 2024-11-16T05:51:26,956 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:26,959 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T05:51:26,963 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T05:51:26,963 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:26,964 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:51:26,964 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T05:51:26,968 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T05:51:26,968 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:26,969 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:51:26,969 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T05:51:26,972 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T05:51:26,972 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:26,973 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:51:26,974 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T05:51:26,974 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T05:51:26,974 INFO [RS:1;3456ee6a3164:44781 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T05:51:26,974 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T05:51:26,977 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T05:51:26,977 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:26,978 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:51:26,979 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T05:51:26,980 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740 2024-11-16T05:51:26,981 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740 2024-11-16T05:51:26,983 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T05:51:26,983 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T05:51:26,984 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T05:51:26,987 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T05:51:26,991 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:51:26,992 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63443389, jitterRate=-0.05461983382701874}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T05:51:26,994 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T05:51:26,996 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731736286956Initializing all the Stores at 1731736286958 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736286959 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736286959Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736286959Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736286959Cleaning up temporary data from old regions at 1731736286983 (+24 ms)Region opened successfully at 1731736286995 (+12 ms) 2024-11-16T05:51:26,996 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T05:51:26,996 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T05:51:26,996 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T05:51:26,996 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T05:51:26,996 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T05:51:26,996 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T05:51:27,000 INFO [RS:0;3456ee6a3164:33007 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T05:51:27,000 INFO [RS:2;3456ee6a3164:43415 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T05:51:27,000 INFO [RS:0;3456ee6a3164:33007 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,000 INFO [RS:2;3456ee6a3164:43415 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,001 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T05:51:27,001 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731736286996Disabling compacts and flushes for region at 1731736286996Disabling writes for close at 1731736286996Writing region close event to WAL at 1731736287000 (+4 ms)Closed at 1731736287000 2024-11-16T05:51:27,001 INFO [RS:1;3456ee6a3164:44781 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T05:51:27,002 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T05:51:27,004 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T05:51:27,004 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-16T05:51:27,005 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T05:51:27,008 INFO [RS:2;3456ee6a3164:43415 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T05:51:27,008 INFO [RS:0;3456ee6a3164:33007 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T05:51:27,010 INFO [RS:2;3456ee6a3164:43415 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,010 INFO [RS:0;3456ee6a3164:33007 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T05:51:27,010 DEBUG [RS:0;3456ee6a3164:33007 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,010 DEBUG [RS:2;3456ee6a3164:43415 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,010 DEBUG [RS:0;3456ee6a3164:33007 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,010 DEBUG [RS:2;3456ee6a3164:43415 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,011 DEBUG [RS:0;3456ee6a3164:33007 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,011 DEBUG [RS:2;3456ee6a3164:43415 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,011 DEBUG [RS:2;3456ee6a3164:43415 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,011 DEBUG [RS:0;3456ee6a3164:33007 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,011 DEBUG [RS:0;3456ee6a3164:33007 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,011 DEBUG [RS:2;3456ee6a3164:43415 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,011 DEBUG [RS:0;3456ee6a3164:33007 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0, corePoolSize=2, maxPoolSize=2 2024-11-16T05:51:27,011 DEBUG [RS:2;3456ee6a3164:43415 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0, corePoolSize=2, maxPoolSize=2 2024-11-16T05:51:27,011 DEBUG [RS:2;3456ee6a3164:43415 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,011 DEBUG [RS:0;3456ee6a3164:33007 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,011 DEBUG [RS:0;3456ee6a3164:33007 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,011 DEBUG [RS:2;3456ee6a3164:43415 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,012 DEBUG [RS:0;3456ee6a3164:33007 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,012 DEBUG [RS:2;3456ee6a3164:43415 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,012 DEBUG [RS:0;3456ee6a3164:33007 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,012 DEBUG [RS:2;3456ee6a3164:43415 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,012 DEBUG [RS:0;3456ee6a3164:33007 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,012 DEBUG [RS:2;3456ee6a3164:43415 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,012 DEBUG [RS:0;3456ee6a3164:33007 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,012 DEBUG [RS:2;3456ee6a3164:43415 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,012 DEBUG [RS:0;3456ee6a3164:33007 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3456ee6a3164:0, corePoolSize=3, maxPoolSize=3 2024-11-16T05:51:27,012 DEBUG [RS:2;3456ee6a3164:43415 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3456ee6a3164:0, corePoolSize=3, maxPoolSize=3 2024-11-16T05:51:27,012 DEBUG [RS:2;3456ee6a3164:43415 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0, corePoolSize=3, maxPoolSize=3 2024-11-16T05:51:27,012 DEBUG [RS:0;3456ee6a3164:33007 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0, corePoolSize=3, maxPoolSize=3 2024-11-16T05:51:27,013 INFO [RS:1;3456ee6a3164:44781 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T05:51:27,013 INFO [RS:1;3456ee6a3164:44781 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,017 INFO [RS:1;3456ee6a3164:44781 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T05:51:27,018 INFO [RS:2;3456ee6a3164:43415 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,018 INFO [RS:2;3456ee6a3164:43415 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,018 INFO [RS:2;3456ee6a3164:43415 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,019 INFO [RS:2;3456ee6a3164:43415 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,019 INFO [RS:2;3456ee6a3164:43415 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,019 INFO [RS:2;3456ee6a3164:43415 {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,43415,1731736285705-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T05:51:27,019 INFO [RS:1;3456ee6a3164:44781 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T05:51:27,019 INFO [RS:1;3456ee6a3164:44781 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,020 DEBUG [RS:1;3456ee6a3164:44781 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,020 DEBUG [RS:1;3456ee6a3164:44781 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,020 DEBUG [RS:1;3456ee6a3164:44781 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,020 DEBUG [RS:1;3456ee6a3164:44781 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,020 DEBUG [RS:1;3456ee6a3164:44781 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,020 DEBUG [RS:1;3456ee6a3164:44781 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0, corePoolSize=2, maxPoolSize=2 2024-11-16T05:51:27,021 DEBUG [RS:1;3456ee6a3164:44781 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,021 DEBUG [RS:1;3456ee6a3164:44781 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,021 DEBUG [RS:1;3456ee6a3164:44781 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,021 DEBUG [RS:1;3456ee6a3164:44781 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,021 DEBUG [RS:1;3456ee6a3164:44781 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,021 DEBUG [RS:1;3456ee6a3164:44781 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3456ee6a3164:0, corePoolSize=1, maxPoolSize=1 2024-11-16T05:51:27,021 DEBUG [RS:1;3456ee6a3164:44781 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3456ee6a3164:0, corePoolSize=3, maxPoolSize=3 2024-11-16T05:51:27,022 DEBUG [RS:1;3456ee6a3164:44781 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3456ee6a3164:0, corePoolSize=3, maxPoolSize=3 2024-11-16T05:51:27,022 INFO [RS:0;3456ee6a3164:33007 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,022 INFO [RS:0;3456ee6a3164:33007 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,022 INFO [RS:0;3456ee6a3164:33007 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,022 INFO [RS:0;3456ee6a3164:33007 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,022 INFO [RS:0;3456ee6a3164:33007 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,023 INFO [RS:0;3456ee6a3164:33007 {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,33007,1731736285465-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T05:51:27,024 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:51:27,028 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-16T05:51:27,031 INFO [RS:1;3456ee6a3164:44781 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,031 INFO [RS:1;3456ee6a3164:44781 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,031 INFO [RS:1;3456ee6a3164:44781 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,031 INFO [RS:1;3456ee6a3164:44781 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,032 INFO [RS:1;3456ee6a3164:44781 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,032 INFO [RS:1;3456ee6a3164:44781 {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,44781,1731736285630-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T05:51:27,049 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T05:51:27,050 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T05:51:27,051 INFO [RS:0;3456ee6a3164:33007 {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,33007,1731736285465-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,051 INFO [RS:2;3456ee6a3164:43415 {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,43415,1731736285705-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,051 INFO [RS:0;3456ee6a3164:33007 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,051 INFO [RS:2;3456ee6a3164:43415 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,052 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.Replication(171): 3456ee6a3164,33007,1731736285465 started 2024-11-16T05:51:27,052 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.Replication(171): 3456ee6a3164,43415,1731736285705 started 2024-11-16T05:51:27,053 INFO [RS:1;3456ee6a3164:44781 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T05:51:27,053 INFO [RS:1;3456ee6a3164:44781 {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,44781,1731736285630-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,053 INFO [RS:1;3456ee6a3164:44781 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,054 INFO [RS:1;3456ee6a3164:44781 {}] regionserver.Replication(171): 3456ee6a3164,44781,1731736285630 started 2024-11-16T05:51:27,072 INFO [RS:1;3456ee6a3164:44781 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,073 INFO [RS:1;3456ee6a3164:44781 {}] regionserver.HRegionServer(1482): Serving as 3456ee6a3164,44781,1731736285630, RpcServer on 3456ee6a3164/172.17.0.2:44781, sessionid=0x100471607020002 2024-11-16T05:51:27,074 DEBUG [RS:1;3456ee6a3164:44781 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T05:51:27,074 DEBUG [RS:1;3456ee6a3164:44781 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3456ee6a3164,44781,1731736285630 2024-11-16T05:51:27,074 DEBUG [RS:1;3456ee6a3164:44781 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3456ee6a3164,44781,1731736285630' 2024-11-16T05:51:27,074 DEBUG [RS:1;3456ee6a3164:44781 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T05:51:27,075 DEBUG [RS:1;3456ee6a3164:44781 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T05:51:27,075 INFO [RS:0;3456ee6a3164:33007 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,075 INFO [RS:2;3456ee6a3164:43415 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,075 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer(1482): Serving as 3456ee6a3164,33007,1731736285465, RpcServer on 3456ee6a3164/172.17.0.2:33007, sessionid=0x100471607020001 2024-11-16T05:51:27,075 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(1482): Serving as 3456ee6a3164,43415,1731736285705, RpcServer on 3456ee6a3164/172.17.0.2:43415, sessionid=0x100471607020003 2024-11-16T05:51:27,076 DEBUG [RS:2;3456ee6a3164:43415 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T05:51:27,076 DEBUG [RS:0;3456ee6a3164:33007 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T05:51:27,076 DEBUG [RS:0;3456ee6a3164:33007 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3456ee6a3164,33007,1731736285465 2024-11-16T05:51:27,076 DEBUG [RS:2;3456ee6a3164:43415 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3456ee6a3164,43415,1731736285705 2024-11-16T05:51:27,076 DEBUG [RS:2;3456ee6a3164:43415 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3456ee6a3164,43415,1731736285705' 2024-11-16T05:51:27,076 DEBUG [RS:0;3456ee6a3164:33007 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3456ee6a3164,33007,1731736285465' 2024-11-16T05:51:27,076 DEBUG [RS:1;3456ee6a3164:44781 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T05:51:27,076 DEBUG [RS:0;3456ee6a3164:33007 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T05:51:27,076 DEBUG [RS:2;3456ee6a3164:43415 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T05:51:27,076 DEBUG [RS:1;3456ee6a3164:44781 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T05:51:27,076 DEBUG [RS:1;3456ee6a3164:44781 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3456ee6a3164,44781,1731736285630 2024-11-16T05:51:27,076 DEBUG [RS:1;3456ee6a3164:44781 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3456ee6a3164,44781,1731736285630' 2024-11-16T05:51:27,076 DEBUG [RS:1;3456ee6a3164:44781 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T05:51:27,077 DEBUG [RS:0;3456ee6a3164:33007 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T05:51:27,077 DEBUG [RS:2;3456ee6a3164:43415 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T05:51:27,077 DEBUG [RS:1;3456ee6a3164:44781 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T05:51:27,077 DEBUG [RS:2;3456ee6a3164:43415 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T05:51:27,077 DEBUG [RS:2;3456ee6a3164:43415 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T05:51:27,077 DEBUG [RS:1;3456ee6a3164:44781 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T05:51:27,077 DEBUG [RS:2;3456ee6a3164:43415 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3456ee6a3164,43415,1731736285705 2024-11-16T05:51:27,077 DEBUG [RS:0;3456ee6a3164:33007 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T05:51:27,077 DEBUG [RS:0;3456ee6a3164:33007 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T05:51:27,077 DEBUG [RS:2;3456ee6a3164:43415 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3456ee6a3164,43415,1731736285705' 2024-11-16T05:51:27,077 INFO [RS:1;3456ee6a3164:44781 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T05:51:27,077 DEBUG [RS:2;3456ee6a3164:43415 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T05:51:27,077 DEBUG [RS:0;3456ee6a3164:33007 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3456ee6a3164,33007,1731736285465 2024-11-16T05:51:27,077 INFO [RS:1;3456ee6a3164:44781 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T05:51:27,078 DEBUG [RS:0;3456ee6a3164:33007 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3456ee6a3164,33007,1731736285465' 2024-11-16T05:51:27,078 DEBUG [RS:0;3456ee6a3164:33007 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T05:51:27,078 DEBUG [RS:2;3456ee6a3164:43415 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T05:51:27,078 DEBUG [RS:0;3456ee6a3164:33007 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T05:51:27,079 DEBUG [RS:0;3456ee6a3164:33007 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T05:51:27,079 INFO [RS:0;3456ee6a3164:33007 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T05:51:27,079 DEBUG [RS:2;3456ee6a3164:43415 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T05:51:27,079 INFO [RS:0;3456ee6a3164:33007 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T05:51:27,079 INFO [RS:2;3456ee6a3164:43415 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T05:51:27,079 INFO [RS:2;3456ee6a3164:43415 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T05:51:27,179 WARN [3456ee6a3164:44685 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-16T05:51:27,184 INFO [RS:1;3456ee6a3164:44781 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T05:51:27,184 INFO [RS:0;3456ee6a3164:33007 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T05:51:27,185 INFO [RS:2;3456ee6a3164:43415 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T05:51:27,188 INFO [RS:0;3456ee6a3164:33007 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3456ee6a3164%2C33007%2C1731736285465, suffix=, logDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,33007,1731736285465, archiveDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/oldWALs, maxLogs=32 2024-11-16T05:51:27,189 INFO [RS:2;3456ee6a3164:43415 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3456ee6a3164%2C43415%2C1731736285705, suffix=, logDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,43415,1731736285705, archiveDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/oldWALs, maxLogs=32 2024-11-16T05:51:27,189 INFO [RS:1;3456ee6a3164:44781 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3456ee6a3164%2C44781%2C1731736285630, suffix=, logDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630, archiveDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/oldWALs, maxLogs=32 2024-11-16T05:51:27,210 DEBUG [RS:2;3456ee6a3164:43415 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,43415,1731736285705/3456ee6a3164%2C43415%2C1731736285705.1731736287193, exclude list is [], retry=0 2024-11-16T05:51:27,217 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33441,DS-f0a76934-b7f3-4998-a613-d00266c6901a,DISK] 2024-11-16T05:51:27,217 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36993,DS-d9cb5fed-ca91-420f-8b4c-9e853c8c3426,DISK] 2024-11-16T05:51:27,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32873,DS-56cc9d96-a128-48c4-add0-03db81c0fd8d,DISK] 2024-11-16T05:51:27,220 DEBUG [RS:0;3456ee6a3164:33007 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,33007,1731736285465/3456ee6a3164%2C33007%2C1731736285465.1731736287193, exclude list is [], retry=0 2024-11-16T05:51:27,224 DEBUG [RS:1;3456ee6a3164:44781 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630/3456ee6a3164%2C44781%2C1731736285630.1731736287193, exclude list is [], retry=0 2024-11-16T05:51:27,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36993,DS-d9cb5fed-ca91-420f-8b4c-9e853c8c3426,DISK] 2024-11-16T05:51:27,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33441,DS-f0a76934-b7f3-4998-a613-d00266c6901a,DISK] 2024-11-16T05:51:27,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32873,DS-56cc9d96-a128-48c4-add0-03db81c0fd8d,DISK] 2024-11-16T05:51:27,233 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36993,DS-d9cb5fed-ca91-420f-8b4c-9e853c8c3426,DISK] 2024-11-16T05:51:27,233 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32873,DS-56cc9d96-a128-48c4-add0-03db81c0fd8d,DISK] 2024-11-16T05:51:27,234 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33441,DS-f0a76934-b7f3-4998-a613-d00266c6901a,DISK] 2024-11-16T05:51:27,234 INFO [RS:2;3456ee6a3164:43415 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,43415,1731736285705/3456ee6a3164%2C43415%2C1731736285705.1731736287193 2024-11-16T05:51:27,239 DEBUG [RS:2;3456ee6a3164:43415 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37155:37155),(127.0.0.1/127.0.0.1:37513:37513),(127.0.0.1/127.0.0.1:43727:43727)] 2024-11-16T05:51:27,250 INFO [RS:0;3456ee6a3164:33007 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,33007,1731736285465/3456ee6a3164%2C33007%2C1731736285465.1731736287193 2024-11-16T05:51:27,250 INFO [RS:1;3456ee6a3164:44781 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630/3456ee6a3164%2C44781%2C1731736285630.1731736287193 2024-11-16T05:51:27,253 DEBUG [RS:0;3456ee6a3164:33007 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37155:37155),(127.0.0.1/127.0.0.1:43727:43727),(127.0.0.1/127.0.0.1:37513:37513)] 2024-11-16T05:51:27,254 DEBUG [RS:1;3456ee6a3164:44781 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37513:37513),(127.0.0.1/127.0.0.1:43727:43727),(127.0.0.1/127.0.0.1:37155:37155)] 2024-11-16T05:51:27,432 DEBUG [3456ee6a3164:44685 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-16T05:51:27,440 DEBUG [3456ee6a3164:44685 {}] balancer.BalancerClusterState(204): Hosts are {3456ee6a3164=0} racks are {/default-rack=0} 2024-11-16T05:51:27,446 DEBUG [3456ee6a3164:44685 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T05:51:27,446 DEBUG [3456ee6a3164:44685 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T05:51:27,446 DEBUG [3456ee6a3164:44685 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-16T05:51:27,446 DEBUG [3456ee6a3164:44685 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T05:51:27,447 DEBUG [3456ee6a3164:44685 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T05:51:27,447 DEBUG [3456ee6a3164:44685 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-16T05:51:27,447 INFO [3456ee6a3164:44685 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T05:51:27,447 INFO [3456ee6a3164:44685 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T05:51:27,447 INFO [3456ee6a3164:44685 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-16T05:51:27,447 DEBUG [3456ee6a3164:44685 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T05:51:27,454 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=3456ee6a3164,44781,1731736285630 2024-11-16T05:51:27,460 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,44781,1731736285630, state=OPENING 2024-11-16T05:51:27,464 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-16T05:51:27,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:51:27,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44781-0x100471607020002, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:51:27,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:51:27,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:51:27,467 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:51:27,467 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:51:27,467 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:51:27,467 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:51:27,469 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:51:27,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,44781,1731736285630}] 2024-11-16T05:51:27,650 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T05:51:27,652 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56689, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T05:51:27,664 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T05:51:27,664 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T05:51:27,665 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T05:51:27,668 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3456ee6a3164%2C44781%2C1731736285630.meta, suffix=.meta, logDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630, archiveDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/oldWALs, maxLogs=32 2024-11-16T05:51:27,685 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630/3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, exclude list is [], retry=0 2024-11-16T05:51:27,689 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32873,DS-56cc9d96-a128-48c4-add0-03db81c0fd8d,DISK] 2024-11-16T05:51:27,689 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36993,DS-d9cb5fed-ca91-420f-8b4c-9e853c8c3426,DISK] 2024-11-16T05:51:27,690 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33441,DS-f0a76934-b7f3-4998-a613-d00266c6901a,DISK] 2024-11-16T05:51:27,694 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630/3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta 2024-11-16T05:51:27,696 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43727:43727),(127.0.0.1/127.0.0.1:37513:37513),(127.0.0.1/127.0.0.1:37155:37155)] 2024-11-16T05:51:27,697 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:51:27,698 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T05:51:27,702 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T05:51:27,707 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T05:51:27,712 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T05:51:27,713 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:27,713 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T05:51:27,713 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T05:51:27,720 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T05:51:27,722 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T05:51:27,722 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:27,723 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:51:27,724 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T05:51:27,725 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T05:51:27,725 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:27,726 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:51:27,726 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T05:51:27,728 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T05:51:27,728 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:27,729 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:51:27,729 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T05:51:27,731 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T05:51:27,731 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:27,732 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:51:27,732 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T05:51:27,733 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740 2024-11-16T05:51:27,736 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740 2024-11-16T05:51:27,739 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T05:51:27,739 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T05:51:27,740 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T05:51:27,744 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T05:51:27,747 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74922981, jitterRate=0.11643941700458527}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T05:51:27,747 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T05:51:27,749 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731736287714Writing region info on filesystem at 1731736287714Initializing all the Stores at 1731736287717 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736287717Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736287720 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736287720Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736287720Cleaning up temporary data from old regions at 1731736287739 (+19 ms)Running coprocessor post-open hooks at 1731736287747 (+8 ms)Region opened successfully at 1731736287749 (+2 ms) 2024-11-16T05:51:27,757 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731736287641 2024-11-16T05:51:27,776 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T05:51:27,777 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T05:51:27,779 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,44781,1731736285630 2024-11-16T05:51:27,785 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,44781,1731736285630, state=OPEN 2024-11-16T05:51:27,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:51:27,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44781-0x100471607020002, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:51:27,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:51:27,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:51:27,789 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:51:27,789 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:51:27,789 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:51:27,789 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:51:27,789 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=3456ee6a3164,44781,1731736285630 2024-11-16T05:51:27,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-16T05:51:27,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,44781,1731736285630 in 319 msec 2024-11-16T05:51:27,804 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-16T05:51:27,804 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 788 msec 2024-11-16T05:51:27,805 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T05:51:27,805 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-16T05:51:27,827 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:51:27,828 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:51:27,878 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:51:27,881 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56235, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:51:27,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1840 sec 2024-11-16T05:51:27,919 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731736287919, completionTime=-1 2024-11-16T05:51:27,923 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-16T05:51:27,923 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:51:27,950 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-16T05:51:27,950 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736347950 2024-11-16T05:51:27,950 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736407950 2024-11-16T05:51:27,950 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-11-16T05:51:27,952 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:51:27,960 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,44685,1731736284655-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,961 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,44685,1731736284655-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,961 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,44685,1731736284655-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,963 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-3456ee6a3164:44685, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,963 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,964 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:27,971 DEBUG [master/3456ee6a3164:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-16T05:51:27,999 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.174sec 2024-11-16T05:51:28,001 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-16T05:51:28,003 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-16T05:51:28,004 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-16T05:51:28,005 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-16T05:51:28,005 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-16T05:51:28,006 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,44685,1731736284655-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T05:51:28,007 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,44685,1731736284655-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-16T05:51:28,014 DEBUG [master/3456ee6a3164:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-16T05:51:28,015 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-16T05:51:28,015 INFO [master/3456ee6a3164:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3456ee6a3164,44685,1731736284655-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T05:51:28,085 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f113c0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:51:28,090 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-16T05:51:28,090 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-16T05:51:28,096 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3456ee6a3164,44685,-1 for getting cluster id 2024-11-16T05:51:28,099 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T05:51:28,108 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '64bcd7a0-b620-4e04-85b3-a19d6690a50b' 2024-11-16T05:51:28,111 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T05:51:28,111 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "64bcd7a0-b620-4e04-85b3-a19d6690a50b" 2024-11-16T05:51:28,111 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3376499a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:51:28,112 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3456ee6a3164,44685,-1] 2024-11-16T05:51:28,115 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T05:51:28,117 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:51:28,118 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57872, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:51:28,121 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4225f0d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T05:51:28,121 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:51:28,128 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:51:28,128 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:51:28,130 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:51:28,151 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=3456ee6a3164,44685,1731736284655 2024-11-16T05:51:28,153 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-16T05:51:28,159 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T05:51:28,164 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 3456ee6a3164,44685,1731736284655 2024-11-16T05:51:28,167 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@57cb2f5b 2024-11-16T05:51:28,168 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T05:51:28,171 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57886, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T05:51:28,178 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44685 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T05:51:28,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44685 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-16T05:51:28,191 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T05:51:28,193 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44685 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true" procId is: 4 2024-11-16T05:51:28,194 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:28,197 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T05:51:28,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44685 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:51:28,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741837_1013 (size=1020) 2024-11-16T05:51:28,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741837_1013 (size=1020) 2024-11-16T05:51:28,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741837_1013 (size=1020) 2024-11-16T05:51:28,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44685 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:51:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44685 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:51:28,625 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b59301af2f6042086ee9455ab281e7e6, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 2024-11-16T05:51:28,626 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 54bd8d95e5df76bdd2472b3a67cd2b8b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731736288173.54bd8d95e5df76bdd2472b3a67cd2b8b.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 2024-11-16T05:51:28,627 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => de75e055385b7b1b204f0aa9c1924d0e, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731736288173.de75e055385b7b1b204f0aa9c1924d0e.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 2024-11-16T05:51:28,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741838_1014 (size=107) 2024-11-16T05:51:28,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741838_1014 (size=107) 2024-11-16T05:51:28,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741838_1014 (size=107) 2024-11-16T05:51:28,652 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:28,652 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing b59301af2f6042086ee9455ab281e7e6, disabling compactions & flushes 2024-11-16T05:51:28,652 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6. 2024-11-16T05:51:28,652 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6. 2024-11-16T05:51:28,652 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6. after waiting 0 ms 2024-11-16T05:51:28,653 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6. 2024-11-16T05:51:28,653 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6. 2024-11-16T05:51:28,653 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for b59301af2f6042086ee9455ab281e7e6: Waiting for close lock at 1731736288652Disabling compacts and flushes for region at 1731736288652Disabling writes for close at 1731736288653 (+1 ms)Writing region close event to WAL at 1731736288653Closed at 1731736288653 2024-11-16T05:51:28,653 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 42e678de61063128349f238929e3d296, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 2024-11-16T05:51:28,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741840_1016 (size=110) 2024-11-16T05:51:28,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741840_1016 (size=110) 2024-11-16T05:51:28,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741840_1016 (size=110) 2024-11-16T05:51:28,662 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731736288173.54bd8d95e5df76bdd2472b3a67cd2b8b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:28,662 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 54bd8d95e5df76bdd2472b3a67cd2b8b, disabling compactions & flushes 2024-11-16T05:51:28,662 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731736288173.54bd8d95e5df76bdd2472b3a67cd2b8b. 2024-11-16T05:51:28,662 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731736288173.54bd8d95e5df76bdd2472b3a67cd2b8b. 2024-11-16T05:51:28,663 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731736288173.54bd8d95e5df76bdd2472b3a67cd2b8b. after waiting 0 ms 2024-11-16T05:51:28,663 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731736288173.54bd8d95e5df76bdd2472b3a67cd2b8b. 2024-11-16T05:51:28,663 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731736288173.54bd8d95e5df76bdd2472b3a67cd2b8b. 2024-11-16T05:51:28,663 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 54bd8d95e5df76bdd2472b3a67cd2b8b: Waiting for close lock at 1731736288662Disabling compacts and flushes for region at 1731736288662Disabling writes for close at 1731736288663 (+1 ms)Writing region close event to WAL at 1731736288663Closed at 1731736288663 2024-11-16T05:51:28,664 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 53faee5a89035a5faf9a235d93b486c9, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731736288173.53faee5a89035a5faf9a235d93b486c9.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 2024-11-16T05:51:28,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741841_1017 (size=110) 2024-11-16T05:51:28,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741841_1017 (size=110) 2024-11-16T05:51:28,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741841_1017 (size=110) 2024-11-16T05:51:28,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741839_1015 (size=110) 2024-11-16T05:51:28,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741839_1015 (size=110) 2024-11-16T05:51:28,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741839_1015 (size=110) 2024-11-16T05:51:28,673 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731736288173.de75e055385b7b1b204f0aa9c1924d0e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:28,673 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:28,673 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 42e678de61063128349f238929e3d296, disabling compactions & flushes 2024-11-16T05:51:28,673 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing de75e055385b7b1b204f0aa9c1924d0e, disabling compactions & flushes 2024-11-16T05:51:28,673 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731736288173.de75e055385b7b1b204f0aa9c1924d0e. 2024-11-16T05:51:28,673 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296. 2024-11-16T05:51:28,673 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731736288173.de75e055385b7b1b204f0aa9c1924d0e. 2024-11-16T05:51:28,673 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296. 2024-11-16T05:51:28,673 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296. after waiting 0 ms 2024-11-16T05:51:28,673 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731736288173.de75e055385b7b1b204f0aa9c1924d0e. after waiting 0 ms 2024-11-16T05:51:28,673 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296. 2024-11-16T05:51:28,673 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731736288173.de75e055385b7b1b204f0aa9c1924d0e. 2024-11-16T05:51:28,673 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296. 2024-11-16T05:51:28,673 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731736288173.de75e055385b7b1b204f0aa9c1924d0e. 2024-11-16T05:51:28,673 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 42e678de61063128349f238929e3d296: Waiting for close lock at 1731736288673Disabling compacts and flushes for region at 1731736288673Disabling writes for close at 1731736288673Writing region close event to WAL at 1731736288673Closed at 1731736288673 2024-11-16T05:51:28,673 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for de75e055385b7b1b204f0aa9c1924d0e: Waiting for close lock at 1731736288673Disabling compacts and flushes for region at 1731736288673Disabling writes for close at 1731736288673Writing region close event to WAL at 1731736288673Closed at 1731736288673 2024-11-16T05:51:28,674 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3e30199c6045096f8f565c93af9b6088, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731736288173.3e30199c6045096f8f565c93af9b6088.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 2024-11-16T05:51:28,674 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 883c672d1e7de6e08e15c0c42278e66d, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 2024-11-16T05:51:28,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741842_1018 (size=110) 2024-11-16T05:51:28,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741842_1018 (size=110) 2024-11-16T05:51:28,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741842_1018 (size=110) 2024-11-16T05:51:28,682 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731736288173.53faee5a89035a5faf9a235d93b486c9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:28,682 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 53faee5a89035a5faf9a235d93b486c9, disabling compactions & flushes 2024-11-16T05:51:28,682 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731736288173.53faee5a89035a5faf9a235d93b486c9. 2024-11-16T05:51:28,682 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731736288173.53faee5a89035a5faf9a235d93b486c9. 2024-11-16T05:51:28,682 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731736288173.53faee5a89035a5faf9a235d93b486c9. after waiting 0 ms 2024-11-16T05:51:28,682 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731736288173.53faee5a89035a5faf9a235d93b486c9. 2024-11-16T05:51:28,682 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731736288173.53faee5a89035a5faf9a235d93b486c9. 2024-11-16T05:51:28,682 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 53faee5a89035a5faf9a235d93b486c9: Waiting for close lock at 1731736288682Disabling compacts and flushes for region at 1731736288682Disabling writes for close at 1731736288682Writing region close event to WAL at 1731736288682Closed at 1731736288682 2024-11-16T05:51:28,683 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => ff61a3918e3c512081ba7b9c1e1a4948, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731736288173.ff61a3918e3c512081ba7b9c1e1a4948.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 2024-11-16T05:51:28,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741844_1020 (size=110) 2024-11-16T05:51:28,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741844_1020 (size=110) 2024-11-16T05:51:28,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741844_1020 (size=110) 2024-11-16T05:51:28,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741843_1019 (size=110) 2024-11-16T05:51:28,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741843_1019 (size=110) 2024-11-16T05:51:28,706 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:28,706 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 883c672d1e7de6e08e15c0c42278e66d, disabling compactions & flushes 2024-11-16T05:51:28,707 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d. 2024-11-16T05:51:28,707 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d. 2024-11-16T05:51:28,707 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d. after waiting 0 ms 2024-11-16T05:51:28,707 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d. 2024-11-16T05:51:28,707 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d. 2024-11-16T05:51:28,707 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 883c672d1e7de6e08e15c0c42278e66d: Waiting for close lock at 1731736288706Disabling compacts and flushes for region at 1731736288706Disabling writes for close at 1731736288707 (+1 ms)Writing region close event to WAL at 1731736288707Closed at 1731736288707 2024-11-16T05:51:28,708 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => c81e1c1af980b616d932866cdd4ff915, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 2024-11-16T05:51:28,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741843_1019 (size=110) 2024-11-16T05:51:28,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741845_1021 (size=110) 2024-11-16T05:51:28,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741845_1021 (size=110) 2024-11-16T05:51:28,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741845_1021 (size=110) 2024-11-16T05:51:28,710 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731736288173.3e30199c6045096f8f565c93af9b6088.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:28,710 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 3e30199c6045096f8f565c93af9b6088, disabling compactions & flushes 2024-11-16T05:51:28,710 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731736288173.3e30199c6045096f8f565c93af9b6088. 2024-11-16T05:51:28,710 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731736288173.3e30199c6045096f8f565c93af9b6088. 2024-11-16T05:51:28,710 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731736288173.3e30199c6045096f8f565c93af9b6088. after waiting 0 ms 2024-11-16T05:51:28,711 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731736288173.3e30199c6045096f8f565c93af9b6088. 2024-11-16T05:51:28,711 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731736288173.3e30199c6045096f8f565c93af9b6088. 2024-11-16T05:51:28,711 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3e30199c6045096f8f565c93af9b6088: Waiting for close lock at 1731736288710Disabling compacts and flushes for region at 1731736288710Disabling writes for close at 1731736288710Writing region close event to WAL at 1731736288711 (+1 ms)Closed at 1731736288711 2024-11-16T05:51:28,711 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5baf9d0bdd8c65c757ad796f93bef122, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731736288173.5baf9d0bdd8c65c757ad796f93bef122.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 2024-11-16T05:51:28,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741847_1023 (size=110) 2024-11-16T05:51:28,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741847_1023 (size=110) 2024-11-16T05:51:28,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741847_1023 (size=110) 2024-11-16T05:51:28,768 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731736288173.5baf9d0bdd8c65c757ad796f93bef122.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:28,769 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 5baf9d0bdd8c65c757ad796f93bef122, disabling compactions & flushes 2024-11-16T05:51:28,769 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731736288173.5baf9d0bdd8c65c757ad796f93bef122. 2024-11-16T05:51:28,769 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731736288173.5baf9d0bdd8c65c757ad796f93bef122. 2024-11-16T05:51:28,769 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731736288173.5baf9d0bdd8c65c757ad796f93bef122. after waiting 0 ms 2024-11-16T05:51:28,769 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731736288173.5baf9d0bdd8c65c757ad796f93bef122. 2024-11-16T05:51:28,769 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731736288173.5baf9d0bdd8c65c757ad796f93bef122. 2024-11-16T05:51:28,769 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5baf9d0bdd8c65c757ad796f93bef122: Waiting for close lock at 1731736288768Disabling compacts and flushes for region at 1731736288768Disabling writes for close at 1731736288769 (+1 ms)Writing region close event to WAL at 1731736288769Closed at 1731736288769 2024-11-16T05:51:28,770 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 63baab6432268e686bc2f01a98a6c75f, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731736288173.63baab6432268e686bc2f01a98a6c75f.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 2024-11-16T05:51:28,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741846_1022 (size=110) 2024-11-16T05:51:28,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741846_1022 (size=110) 2024-11-16T05:51:28,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741846_1022 (size=110) 2024-11-16T05:51:28,798 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:28,798 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing c81e1c1af980b616d932866cdd4ff915, disabling compactions & flushes 2024-11-16T05:51:28,798 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915. 2024-11-16T05:51:28,798 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915. 2024-11-16T05:51:28,798 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915. after waiting 0 ms 2024-11-16T05:51:28,799 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915. 2024-11-16T05:51:28,799 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915. 2024-11-16T05:51:28,799 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for c81e1c1af980b616d932866cdd4ff915: Waiting for close lock at 1731736288798Disabling compacts and flushes for region at 1731736288798Disabling writes for close at 1731736288798Writing region close event to WAL at 1731736288799 (+1 ms)Closed at 1731736288799 2024-11-16T05:51:28,799 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 408fc7812ce9cee6ea99428f155ce5a8, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731736288173.408fc7812ce9cee6ea99428f155ce5a8.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 2024-11-16T05:51:28,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741848_1024 (size=110) 2024-11-16T05:51:28,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741848_1024 (size=110) 2024-11-16T05:51:28,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741848_1024 (size=110) 2024-11-16T05:51:28,814 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731736288173.63baab6432268e686bc2f01a98a6c75f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:28,815 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 63baab6432268e686bc2f01a98a6c75f, disabling compactions & flushes 2024-11-16T05:51:28,815 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731736288173.63baab6432268e686bc2f01a98a6c75f. 2024-11-16T05:51:28,815 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731736288173.63baab6432268e686bc2f01a98a6c75f. 2024-11-16T05:51:28,815 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731736288173.63baab6432268e686bc2f01a98a6c75f. after waiting 0 ms 2024-11-16T05:51:28,815 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731736288173.63baab6432268e686bc2f01a98a6c75f. 2024-11-16T05:51:28,815 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731736288173.63baab6432268e686bc2f01a98a6c75f. 2024-11-16T05:51:28,815 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 63baab6432268e686bc2f01a98a6c75f: Waiting for close lock at 1731736288815Disabling compacts and flushes for region at 1731736288815Disabling writes for close at 1731736288815Writing region close event to WAL at 1731736288815Closed at 1731736288815 2024-11-16T05:51:28,816 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bd79de8195c0452a334d7da789e8f23b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 2024-11-16T05:51:28,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44685 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:51:28,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741849_1025 (size=110) 2024-11-16T05:51:28,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741849_1025 (size=110) 2024-11-16T05:51:28,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741849_1025 (size=110) 2024-11-16T05:51:28,843 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731736288173.408fc7812ce9cee6ea99428f155ce5a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:28,843 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 408fc7812ce9cee6ea99428f155ce5a8, disabling compactions & flushes 2024-11-16T05:51:28,843 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731736288173.408fc7812ce9cee6ea99428f155ce5a8. 2024-11-16T05:51:28,843 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731736288173.408fc7812ce9cee6ea99428f155ce5a8. 2024-11-16T05:51:28,843 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731736288173.408fc7812ce9cee6ea99428f155ce5a8. after waiting 0 ms 2024-11-16T05:51:28,844 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731736288173.408fc7812ce9cee6ea99428f155ce5a8. 2024-11-16T05:51:28,844 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731736288173.408fc7812ce9cee6ea99428f155ce5a8. 2024-11-16T05:51:28,844 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 408fc7812ce9cee6ea99428f155ce5a8: Waiting for close lock at 1731736288843Disabling compacts and flushes for region at 1731736288843Disabling writes for close at 1731736288843Writing region close event to WAL at 1731736288844 (+1 ms)Closed at 1731736288844 2024-11-16T05:51:28,845 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 12c6ab39e16284ff5b8ffeb36e672416, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731736288173.12c6ab39e16284ff5b8ffeb36e672416.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 2024-11-16T05:51:28,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741850_1026 (size=110) 2024-11-16T05:51:28,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741850_1026 (size=110) 2024-11-16T05:51:28,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741850_1026 (size=110) 2024-11-16T05:51:28,855 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:28,855 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing bd79de8195c0452a334d7da789e8f23b, disabling compactions & flushes 2024-11-16T05:51:28,855 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b. 2024-11-16T05:51:28,855 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b. 2024-11-16T05:51:28,856 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b. after waiting 0 ms 2024-11-16T05:51:28,856 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b. 2024-11-16T05:51:28,856 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b. 2024-11-16T05:51:28,856 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for bd79de8195c0452a334d7da789e8f23b: Waiting for close lock at 1731736288855Disabling compacts and flushes for region at 1731736288855Disabling writes for close at 1731736288856 (+1 ms)Writing region close event to WAL at 1731736288856Closed at 1731736288856 2024-11-16T05:51:28,857 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 07f1b4025bc2e2627b7dc3f7c79afd8e, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731736288173.07f1b4025bc2e2627b7dc3f7c79afd8e.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 2024-11-16T05:51:28,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741851_1027 (size=110) 2024-11-16T05:51:28,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741851_1027 (size=110) 2024-11-16T05:51:28,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741851_1027 (size=110) 2024-11-16T05:51:28,879 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731736288173.12c6ab39e16284ff5b8ffeb36e672416.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:28,880 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 12c6ab39e16284ff5b8ffeb36e672416, disabling compactions & flushes 2024-11-16T05:51:28,880 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731736288173.12c6ab39e16284ff5b8ffeb36e672416. 2024-11-16T05:51:28,880 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731736288173.12c6ab39e16284ff5b8ffeb36e672416. 2024-11-16T05:51:28,880 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731736288173.12c6ab39e16284ff5b8ffeb36e672416. after waiting 0 ms 2024-11-16T05:51:28,880 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731736288173.12c6ab39e16284ff5b8ffeb36e672416. 2024-11-16T05:51:28,880 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731736288173.12c6ab39e16284ff5b8ffeb36e672416. 2024-11-16T05:51:28,880 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 12c6ab39e16284ff5b8ffeb36e672416: Waiting for close lock at 1731736288879Disabling compacts and flushes for region at 1731736288879Disabling writes for close at 1731736288880 (+1 ms)Writing region close event to WAL at 1731736288880Closed at 1731736288880 2024-11-16T05:51:28,880 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => ec037e054803eda72bad2d81c6143b00, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 2024-11-16T05:51:28,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741852_1028 (size=110) 2024-11-16T05:51:28,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741852_1028 (size=110) 2024-11-16T05:51:28,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741852_1028 (size=110) 2024-11-16T05:51:28,892 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731736288173.07f1b4025bc2e2627b7dc3f7c79afd8e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:28,892 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 07f1b4025bc2e2627b7dc3f7c79afd8e, disabling compactions & flushes 2024-11-16T05:51:28,892 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731736288173.07f1b4025bc2e2627b7dc3f7c79afd8e. 2024-11-16T05:51:28,892 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731736288173.07f1b4025bc2e2627b7dc3f7c79afd8e. 2024-11-16T05:51:28,892 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731736288173.07f1b4025bc2e2627b7dc3f7c79afd8e. after waiting 0 ms 2024-11-16T05:51:28,892 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731736288173.07f1b4025bc2e2627b7dc3f7c79afd8e. 2024-11-16T05:51:28,892 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731736288173.07f1b4025bc2e2627b7dc3f7c79afd8e. 2024-11-16T05:51:28,892 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 07f1b4025bc2e2627b7dc3f7c79afd8e: Waiting for close lock at 1731736288892Disabling compacts and flushes for region at 1731736288892Disabling writes for close at 1731736288892Writing region close event to WAL at 1731736288892Closed at 1731736288892 2024-11-16T05:51:28,893 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0166203610c1853876df2910d136e997, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731736288173.0166203610c1853876df2910d136e997.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 2024-11-16T05:51:28,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741853_1029 (size=110) 2024-11-16T05:51:28,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741853_1029 (size=110) 2024-11-16T05:51:28,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741853_1029 (size=110) 2024-11-16T05:51:28,915 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:28,916 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing ec037e054803eda72bad2d81c6143b00, disabling compactions & flushes 2024-11-16T05:51:28,916 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00. 2024-11-16T05:51:28,916 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00. 2024-11-16T05:51:28,916 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00. after waiting 0 ms 2024-11-16T05:51:28,916 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00. 2024-11-16T05:51:28,916 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00. 2024-11-16T05:51:28,916 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for ec037e054803eda72bad2d81c6143b00: Waiting for close lock at 1731736288916Disabling compacts and flushes for region at 1731736288916Disabling writes for close at 1731736288916Writing region close event to WAL at 1731736288916Closed at 1731736288916 2024-11-16T05:51:28,917 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => df4b7c945ebb852a560a10e20c40c386, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731736288173.df4b7c945ebb852a560a10e20c40c386.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 2024-11-16T05:51:28,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741854_1030 (size=110) 2024-11-16T05:51:28,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741854_1030 (size=110) 2024-11-16T05:51:28,922 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731736288173.0166203610c1853876df2910d136e997.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:28,922 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 0166203610c1853876df2910d136e997, disabling compactions & flushes 2024-11-16T05:51:28,922 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731736288173.0166203610c1853876df2910d136e997. 2024-11-16T05:51:28,922 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731736288173.0166203610c1853876df2910d136e997. 2024-11-16T05:51:28,922 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731736288173.0166203610c1853876df2910d136e997. after waiting 0 ms 2024-11-16T05:51:28,922 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731736288173.0166203610c1853876df2910d136e997. 2024-11-16T05:51:28,922 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731736288173.0166203610c1853876df2910d136e997. 2024-11-16T05:51:28,922 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0166203610c1853876df2910d136e997: Waiting for close lock at 1731736288922Disabling compacts and flushes for region at 1731736288922Disabling writes for close at 1731736288922Writing region close event to WAL at 1731736288922Closed at 1731736288922 2024-11-16T05:51:28,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741854_1030 (size=110) 2024-11-16T05:51:28,926 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a6c8ea5676b855dac5a939ace6afadb0, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 2024-11-16T05:51:28,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741855_1031 (size=110) 2024-11-16T05:51:28,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741855_1031 (size=110) 2024-11-16T05:51:28,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741855_1031 (size=110) 2024-11-16T05:51:28,960 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731736288173.df4b7c945ebb852a560a10e20c40c386.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:28,961 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing df4b7c945ebb852a560a10e20c40c386, disabling compactions & flushes 2024-11-16T05:51:28,961 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731736288173.df4b7c945ebb852a560a10e20c40c386. 2024-11-16T05:51:28,961 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731736288173.df4b7c945ebb852a560a10e20c40c386. 2024-11-16T05:51:28,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741856_1032 (size=110) 2024-11-16T05:51:28,961 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731736288173.df4b7c945ebb852a560a10e20c40c386. after waiting 0 ms 2024-11-16T05:51:28,961 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731736288173.df4b7c945ebb852a560a10e20c40c386. 2024-11-16T05:51:28,961 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731736288173.df4b7c945ebb852a560a10e20c40c386. 2024-11-16T05:51:28,961 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for df4b7c945ebb852a560a10e20c40c386: Waiting for close lock at 1731736288961Disabling compacts and flushes for region at 1731736288961Disabling writes for close at 1731736288961Writing region close event to WAL at 1731736288961Closed at 1731736288961 2024-11-16T05:51:28,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741856_1032 (size=110) 2024-11-16T05:51:28,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741856_1032 (size=110) 2024-11-16T05:51:28,962 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 2beaf8caee4e552158f6bb091a0437bb, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731736288173.2beaf8caee4e552158f6bb091a0437bb.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 2024-11-16T05:51:28,969 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:28,969 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing a6c8ea5676b855dac5a939ace6afadb0, disabling compactions & flushes 2024-11-16T05:51:28,969 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0. 2024-11-16T05:51:28,969 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0. 2024-11-16T05:51:28,969 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0. after waiting 0 ms 2024-11-16T05:51:28,969 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0. 2024-11-16T05:51:28,969 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0. 2024-11-16T05:51:28,969 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for a6c8ea5676b855dac5a939ace6afadb0: Waiting for close lock at 1731736288969Disabling compacts and flushes for region at 1731736288969Disabling writes for close at 1731736288969Writing region close event to WAL at 1731736288969Closed at 1731736288969 2024-11-16T05:51:28,970 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9df9b8a2b43462b728e2352478a97347, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731736288173.9df9b8a2b43462b728e2352478a97347.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 2024-11-16T05:51:28,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741858_1034 (size=110) 2024-11-16T05:51:28,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741858_1034 (size=110) 2024-11-16T05:51:28,996 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731736288173.9df9b8a2b43462b728e2352478a97347.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:28,997 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 9df9b8a2b43462b728e2352478a97347, disabling compactions & flushes 2024-11-16T05:51:28,997 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731736288173.9df9b8a2b43462b728e2352478a97347. 2024-11-16T05:51:28,997 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731736288173.9df9b8a2b43462b728e2352478a97347. 2024-11-16T05:51:28,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741858_1034 (size=110) 2024-11-16T05:51:28,997 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731736288173.9df9b8a2b43462b728e2352478a97347. after waiting 0 ms 2024-11-16T05:51:28,997 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731736288173.9df9b8a2b43462b728e2352478a97347. 2024-11-16T05:51:28,997 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731736288173.9df9b8a2b43462b728e2352478a97347. 2024-11-16T05:51:28,997 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9df9b8a2b43462b728e2352478a97347: Waiting for close lock at 1731736288997Disabling compacts and flushes for region at 1731736288997Disabling writes for close at 1731736288997Writing region close event to WAL at 1731736288997Closed at 1731736288997 2024-11-16T05:51:28,998 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 10a3e438f5882d261cd799e4d6b75095, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 2024-11-16T05:51:29,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741857_1033 (size=110) 2024-11-16T05:51:29,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741857_1033 (size=110) 2024-11-16T05:51:29,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741857_1033 (size=110) 2024-11-16T05:51:29,010 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731736288173.2beaf8caee4e552158f6bb091a0437bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:29,010 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 2beaf8caee4e552158f6bb091a0437bb, disabling compactions & flushes 2024-11-16T05:51:29,011 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731736288173.2beaf8caee4e552158f6bb091a0437bb. 2024-11-16T05:51:29,011 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731736288173.2beaf8caee4e552158f6bb091a0437bb. 2024-11-16T05:51:29,011 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731736288173.2beaf8caee4e552158f6bb091a0437bb. after waiting 0 ms 2024-11-16T05:51:29,011 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731736288173.2beaf8caee4e552158f6bb091a0437bb. 2024-11-16T05:51:29,011 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731736288173.2beaf8caee4e552158f6bb091a0437bb. 2024-11-16T05:51:29,011 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 2beaf8caee4e552158f6bb091a0437bb: Waiting for close lock at 1731736289010Disabling compacts and flushes for region at 1731736289010Disabling writes for close at 1731736289011 (+1 ms)Writing region close event to WAL at 1731736289011Closed at 1731736289011 2024-11-16T05:51:29,012 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => eb73dbcec883aba378443bd58f1eb724, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731736288173.eb73dbcec883aba378443bd58f1eb724.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 2024-11-16T05:51:29,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741859_1035 (size=110) 2024-11-16T05:51:29,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741859_1035 (size=110) 2024-11-16T05:51:29,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741859_1035 (size=110) 2024-11-16T05:51:29,070 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:29,070 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 10a3e438f5882d261cd799e4d6b75095, disabling compactions & flushes 2024-11-16T05:51:29,070 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095. 2024-11-16T05:51:29,070 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095. 2024-11-16T05:51:29,070 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095. after waiting 0 ms 2024-11-16T05:51:29,071 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095. 2024-11-16T05:51:29,071 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095. 2024-11-16T05:51:29,071 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 10a3e438f5882d261cd799e4d6b75095: Waiting for close lock at 1731736289070Disabling compacts and flushes for region at 1731736289070Disabling writes for close at 1731736289070Writing region close event to WAL at 1731736289071 (+1 ms)Closed at 1731736289071 2024-11-16T05:51:29,072 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ac83f7461cc15e741d0440c6ee2781db, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 2024-11-16T05:51:29,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741860_1036 (size=110) 2024-11-16T05:51:29,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741860_1036 (size=110) 2024-11-16T05:51:29,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741860_1036 (size=110) 2024-11-16T05:51:29,101 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731736288173.eb73dbcec883aba378443bd58f1eb724.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:29,101 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing eb73dbcec883aba378443bd58f1eb724, disabling compactions & flushes 2024-11-16T05:51:29,101 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731736288173.eb73dbcec883aba378443bd58f1eb724. 2024-11-16T05:51:29,102 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731736288173.eb73dbcec883aba378443bd58f1eb724. 2024-11-16T05:51:29,102 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731736288173.eb73dbcec883aba378443bd58f1eb724. after waiting 0 ms 2024-11-16T05:51:29,102 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731736288173.eb73dbcec883aba378443bd58f1eb724. 2024-11-16T05:51:29,102 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731736288173.eb73dbcec883aba378443bd58f1eb724. 2024-11-16T05:51:29,102 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for eb73dbcec883aba378443bd58f1eb724: Waiting for close lock at 1731736289101Disabling compacts and flushes for region at 1731736289101Disabling writes for close at 1731736289102 (+1 ms)Writing region close event to WAL at 1731736289102Closed at 1731736289102 2024-11-16T05:51:29,103 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => afd4d1a884eb847b35b730a1b7a90e0d, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731736288173.afd4d1a884eb847b35b730a1b7a90e0d.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 2024-11-16T05:51:29,111 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731736288173.ff61a3918e3c512081ba7b9c1e1a4948.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:29,112 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing ff61a3918e3c512081ba7b9c1e1a4948, disabling compactions & flushes 2024-11-16T05:51:29,112 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731736288173.ff61a3918e3c512081ba7b9c1e1a4948. 2024-11-16T05:51:29,112 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731736288173.ff61a3918e3c512081ba7b9c1e1a4948. 2024-11-16T05:51:29,112 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731736288173.ff61a3918e3c512081ba7b9c1e1a4948. after waiting 0 ms 2024-11-16T05:51:29,112 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731736288173.ff61a3918e3c512081ba7b9c1e1a4948. 2024-11-16T05:51:29,112 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731736288173.ff61a3918e3c512081ba7b9c1e1a4948. 2024-11-16T05:51:29,112 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for ff61a3918e3c512081ba7b9c1e1a4948: Waiting for close lock at 1731736289112Disabling compacts and flushes for region at 1731736289112Disabling writes for close at 1731736289112Writing region close event to WAL at 1731736289112Closed at 1731736289112 2024-11-16T05:51:29,113 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => ca14297d921e6ca3fee0822ad2bb732e, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731736288173.ca14297d921e6ca3fee0822ad2bb732e.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8 2024-11-16T05:51:29,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741861_1037 (size=110) 2024-11-16T05:51:29,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741861_1037 (size=110) 2024-11-16T05:51:29,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741861_1037 (size=110) 2024-11-16T05:51:29,156 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:29,156 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing ac83f7461cc15e741d0440c6ee2781db, disabling compactions & flushes 2024-11-16T05:51:29,157 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db. 2024-11-16T05:51:29,157 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db. 2024-11-16T05:51:29,157 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db. after waiting 0 ms 2024-11-16T05:51:29,157 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db. 2024-11-16T05:51:29,157 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db. 2024-11-16T05:51:29,157 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for ac83f7461cc15e741d0440c6ee2781db: Waiting for close lock at 1731736289156Disabling compacts and flushes for region at 1731736289156Disabling writes for close at 1731736289157 (+1 ms)Writing region close event to WAL at 1731736289157Closed at 1731736289157 2024-11-16T05:51:29,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741863_1039 (size=107) 2024-11-16T05:51:29,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741863_1039 (size=107) 2024-11-16T05:51:29,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741863_1039 (size=107) 2024-11-16T05:51:29,179 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731736288173.ca14297d921e6ca3fee0822ad2bb732e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:29,179 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing ca14297d921e6ca3fee0822ad2bb732e, disabling compactions & flushes 2024-11-16T05:51:29,179 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731736288173.ca14297d921e6ca3fee0822ad2bb732e. 2024-11-16T05:51:29,179 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731736288173.ca14297d921e6ca3fee0822ad2bb732e. 2024-11-16T05:51:29,179 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731736288173.ca14297d921e6ca3fee0822ad2bb732e. after waiting 0 ms 2024-11-16T05:51:29,179 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731736288173.ca14297d921e6ca3fee0822ad2bb732e. 2024-11-16T05:51:29,179 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731736288173.ca14297d921e6ca3fee0822ad2bb732e. 2024-11-16T05:51:29,180 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for ca14297d921e6ca3fee0822ad2bb732e: Waiting for close lock at 1731736289179Disabling compacts and flushes for region at 1731736289179Disabling writes for close at 1731736289179Writing region close event to WAL at 1731736289179Closed at 1731736289179 2024-11-16T05:51:29,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741862_1038 (size=110) 2024-11-16T05:51:29,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741862_1038 (size=110) 2024-11-16T05:51:29,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741862_1038 (size=110) 2024-11-16T05:51:29,198 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731736288173.afd4d1a884eb847b35b730a1b7a90e0d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:29,198 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing afd4d1a884eb847b35b730a1b7a90e0d, disabling compactions & flushes 2024-11-16T05:51:29,198 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731736288173.afd4d1a884eb847b35b730a1b7a90e0d. 2024-11-16T05:51:29,198 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731736288173.afd4d1a884eb847b35b730a1b7a90e0d. 2024-11-16T05:51:29,198 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731736288173.afd4d1a884eb847b35b730a1b7a90e0d. after waiting 0 ms 2024-11-16T05:51:29,198 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731736288173.afd4d1a884eb847b35b730a1b7a90e0d. 2024-11-16T05:51:29,198 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731736288173.afd4d1a884eb847b35b730a1b7a90e0d. 2024-11-16T05:51:29,198 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for afd4d1a884eb847b35b730a1b7a90e0d: Waiting for close lock at 1731736289198Disabling compacts and flushes for region at 1731736289198Disabling writes for close at 1731736289198Writing region close event to WAL at 1731736289198Closed at 1731736289198 2024-11-16T05:51:29,201 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T05:51:29,206 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6.","families":{"info":[{"qualifier":"regioninfo","vlen":106,"tag":[],"timestamp":"1731736289201"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736289201"}]},"ts":"1731736289201"} 2024-11-16T05:51:29,206 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731736288173.54bd8d95e5df76bdd2472b3a67cd2b8b.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731736289201"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736289201"}]},"ts":"1731736289201"} 2024-11-16T05:51:29,206 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731736289201"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736289201"}]},"ts":"1731736289201"} 2024-11-16T05:51:29,206 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731736288173.de75e055385b7b1b204f0aa9c1924d0e.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731736289201"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736289201"}]},"ts":"1731736289201"} 2024-11-16T05:51:29,207 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731736288173.53faee5a89035a5faf9a235d93b486c9.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731736289201"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736289201"}]},"ts":"1731736289201"} 2024-11-16T05:51:29,207 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731736289201"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736289201"}]},"ts":"1731736289201"} 2024-11-16T05:51:29,207 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731736288173.3e30199c6045096f8f565c93af9b6088.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731736289201"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736289201"}]},"ts":"1731736289201"} 2024-11-16T05:51:29,207 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731736288173.5baf9d0bdd8c65c757ad796f93bef122.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731736289201"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736289201"}]},"ts":"1731736289201"} 2024-11-16T05:51:29,207 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731736289201"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736289201"}]},"ts":"1731736289201"} 2024-11-16T05:51:29,208 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731736288173.63baab6432268e686bc2f01a98a6c75f.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731736289201"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736289201"}]},"ts":"1731736289201"} 2024-11-16T05:51:29,208 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731736288173.408fc7812ce9cee6ea99428f155ce5a8.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731736289201"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736289201"}]},"ts":"1731736289201"} 2024-11-16T05:51:29,208 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731736289201"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736289201"}]},"ts":"1731736289201"} 2024-11-16T05:51:29,208 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731736288173.12c6ab39e16284ff5b8ffeb36e672416.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731736289201"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736289201"}]},"ts":"1731736289201"} 2024-11-16T05:51:29,208 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731736288173.07f1b4025bc2e2627b7dc3f7c79afd8e.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731736289201"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736289201"}]},"ts":"1731736289201"} 2024-11-16T05:51:29,208 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731736289201"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736289201"}]},"ts":"1731736289201"} 2024-11-16T05:51:29,208 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731736288173.0166203610c1853876df2910d136e997.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731736289201"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736289201"}]},"ts":"1731736289201"} 2024-11-16T05:51:29,209 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731736288173.df4b7c945ebb852a560a10e20c40c386.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731736289201"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736289201"}]},"ts":"1731736289201"} 2024-11-16T05:51:29,209 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731736289201"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736289201"}]},"ts":"1731736289201"} 2024-11-16T05:51:29,209 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731736288173.9df9b8a2b43462b728e2352478a97347.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731736289201"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736289201"}]},"ts":"1731736289201"} 2024-11-16T05:51:29,209 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731736288173.2beaf8caee4e552158f6bb091a0437bb.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731736289201"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736289201"}]},"ts":"1731736289201"} 2024-11-16T05:51:29,209 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731736289201"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736289201"}]},"ts":"1731736289201"} 2024-11-16T05:51:29,209 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731736288173.eb73dbcec883aba378443bd58f1eb724.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731736289201"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736289201"}]},"ts":"1731736289201"} 2024-11-16T05:51:29,209 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731736288173.ff61a3918e3c512081ba7b9c1e1a4948.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731736289201"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736289201"}]},"ts":"1731736289201"} 2024-11-16T05:51:29,210 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731736289201"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736289201"}]},"ts":"1731736289201"} 2024-11-16T05:51:29,210 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731736288173.ca14297d921e6ca3fee0822ad2bb732e.","families":{"info":[{"qualifier":"regioninfo","vlen":106,"tag":[],"timestamp":"1731736289201"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736289201"}]},"ts":"1731736289201"} 2024-11-16T05:51:29,210 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731736288173.afd4d1a884eb847b35b730a1b7a90e0d.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731736289201"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731736289201"}]},"ts":"1731736289201"} 2024-11-16T05:51:29,254 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-16T05:51:29,257 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T05:51:29,259 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736289257"}]},"ts":"1731736289257"} 2024-11-16T05:51:29,264 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, state=ENABLING in hbase:meta 2024-11-16T05:51:29,265 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {3456ee6a3164=0} racks are {/default-rack=0} 2024-11-16T05:51:29,312 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T05:51:29,312 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T05:51:29,312 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-16T05:51:29,312 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T05:51:29,313 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T05:51:29,313 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-16T05:51:29,313 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T05:51:29,313 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T05:51:29,313 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-16T05:51:29,313 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T05:51:29,317 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b59301af2f6042086ee9455ab281e7e6, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=54bd8d95e5df76bdd2472b3a67cd2b8b, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=de75e055385b7b1b204f0aa9c1924d0e, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=42e678de61063128349f238929e3d296, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=53faee5a89035a5faf9a235d93b486c9, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3e30199c6045096f8f565c93af9b6088, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=883c672d1e7de6e08e15c0c42278e66d, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ff61a3918e3c512081ba7b9c1e1a4948, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c81e1c1af980b616d932866cdd4ff915, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5baf9d0bdd8c65c757ad796f93bef122, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=63baab6432268e686bc2f01a98a6c75f, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=408fc7812ce9cee6ea99428f155ce5a8, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bd79de8195c0452a334d7da789e8f23b, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=12c6ab39e16284ff5b8ffeb36e672416, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=07f1b4025bc2e2627b7dc3f7c79afd8e, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ec037e054803eda72bad2d81c6143b00, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0166203610c1853876df2910d136e997, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=df4b7c945ebb852a560a10e20c40c386, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a6c8ea5676b855dac5a939ace6afadb0, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2beaf8caee4e552158f6bb091a0437bb, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9df9b8a2b43462b728e2352478a97347, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=10a3e438f5882d261cd799e4d6b75095, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=eb73dbcec883aba378443bd58f1eb724, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ac83f7461cc15e741d0440c6ee2781db, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=afd4d1a884eb847b35b730a1b7a90e0d, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ca14297d921e6ca3fee0822ad2bb732e, ASSIGN}] 2024-11-16T05:51:29,331 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ca14297d921e6ca3fee0822ad2bb732e, ASSIGN 2024-11-16T05:51:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44685 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:51:29,335 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ca14297d921e6ca3fee0822ad2bb732e, ASSIGN; state=OFFLINE, location=3456ee6a3164,33007,1731736285465; forceNewPlan=false, retain=false 2024-11-16T05:51:29,338 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=afd4d1a884eb847b35b730a1b7a90e0d, ASSIGN 2024-11-16T05:51:29,341 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=afd4d1a884eb847b35b730a1b7a90e0d, ASSIGN; state=OFFLINE, location=3456ee6a3164,43415,1731736285705; forceNewPlan=false, retain=false 2024-11-16T05:51:29,352 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ac83f7461cc15e741d0440c6ee2781db, ASSIGN 2024-11-16T05:51:29,355 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ac83f7461cc15e741d0440c6ee2781db, ASSIGN; state=OFFLINE, location=3456ee6a3164,44781,1731736285630; forceNewPlan=false, retain=false 2024-11-16T05:51:29,357 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=eb73dbcec883aba378443bd58f1eb724, ASSIGN 2024-11-16T05:51:29,359 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=eb73dbcec883aba378443bd58f1eb724, ASSIGN; state=OFFLINE, location=3456ee6a3164,33007,1731736285465; forceNewPlan=false, retain=false 2024-11-16T05:51:29,361 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=10a3e438f5882d261cd799e4d6b75095, ASSIGN 2024-11-16T05:51:29,363 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=10a3e438f5882d261cd799e4d6b75095, ASSIGN; state=OFFLINE, location=3456ee6a3164,44781,1731736285630; forceNewPlan=false, retain=false 2024-11-16T05:51:29,364 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9df9b8a2b43462b728e2352478a97347, ASSIGN 2024-11-16T05:51:29,366 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9df9b8a2b43462b728e2352478a97347, ASSIGN; state=OFFLINE, location=3456ee6a3164,33007,1731736285465; forceNewPlan=false, retain=false 2024-11-16T05:51:29,368 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2beaf8caee4e552158f6bb091a0437bb, ASSIGN 2024-11-16T05:51:29,369 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2beaf8caee4e552158f6bb091a0437bb, ASSIGN; state=OFFLINE, location=3456ee6a3164,43415,1731736285705; forceNewPlan=false, retain=false 2024-11-16T05:51:29,371 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a6c8ea5676b855dac5a939ace6afadb0, ASSIGN 2024-11-16T05:51:29,373 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a6c8ea5676b855dac5a939ace6afadb0, ASSIGN; state=OFFLINE, location=3456ee6a3164,44781,1731736285630; forceNewPlan=false, retain=false 2024-11-16T05:51:29,374 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=df4b7c945ebb852a560a10e20c40c386, ASSIGN 2024-11-16T05:51:29,376 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=df4b7c945ebb852a560a10e20c40c386, ASSIGN; state=OFFLINE, location=3456ee6a3164,43415,1731736285705; forceNewPlan=false, retain=false 2024-11-16T05:51:29,378 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0166203610c1853876df2910d136e997, ASSIGN 2024-11-16T05:51:29,379 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0166203610c1853876df2910d136e997, ASSIGN; state=OFFLINE, location=3456ee6a3164,33007,1731736285465; forceNewPlan=false, retain=false 2024-11-16T05:51:29,381 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ec037e054803eda72bad2d81c6143b00, ASSIGN 2024-11-16T05:51:29,383 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ec037e054803eda72bad2d81c6143b00, ASSIGN; state=OFFLINE, location=3456ee6a3164,44781,1731736285630; forceNewPlan=false, retain=false 2024-11-16T05:51:29,384 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=07f1b4025bc2e2627b7dc3f7c79afd8e, ASSIGN 2024-11-16T05:51:29,385 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=07f1b4025bc2e2627b7dc3f7c79afd8e, ASSIGN; state=OFFLINE, location=3456ee6a3164,43415,1731736285705; forceNewPlan=false, retain=false 2024-11-16T05:51:29,387 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=12c6ab39e16284ff5b8ffeb36e672416, ASSIGN 2024-11-16T05:51:29,388 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=12c6ab39e16284ff5b8ffeb36e672416, ASSIGN; state=OFFLINE, location=3456ee6a3164,33007,1731736285465; forceNewPlan=false, retain=false 2024-11-16T05:51:29,389 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bd79de8195c0452a334d7da789e8f23b, ASSIGN 2024-11-16T05:51:29,391 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bd79de8195c0452a334d7da789e8f23b, ASSIGN; state=OFFLINE, location=3456ee6a3164,44781,1731736285630; forceNewPlan=false, retain=false 2024-11-16T05:51:29,392 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=408fc7812ce9cee6ea99428f155ce5a8, ASSIGN 2024-11-16T05:51:29,393 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=408fc7812ce9cee6ea99428f155ce5a8, ASSIGN; state=OFFLINE, location=3456ee6a3164,43415,1731736285705; forceNewPlan=false, retain=false 2024-11-16T05:51:29,394 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=63baab6432268e686bc2f01a98a6c75f, ASSIGN 2024-11-16T05:51:29,395 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=63baab6432268e686bc2f01a98a6c75f, ASSIGN; state=OFFLINE, location=3456ee6a3164,33007,1731736285465; forceNewPlan=false, retain=false 2024-11-16T05:51:29,397 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5baf9d0bdd8c65c757ad796f93bef122, ASSIGN 2024-11-16T05:51:29,398 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5baf9d0bdd8c65c757ad796f93bef122, ASSIGN; state=OFFLINE, location=3456ee6a3164,43415,1731736285705; forceNewPlan=false, retain=false 2024-11-16T05:51:29,399 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c81e1c1af980b616d932866cdd4ff915, ASSIGN 2024-11-16T05:51:29,401 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c81e1c1af980b616d932866cdd4ff915, ASSIGN; state=OFFLINE, location=3456ee6a3164,44781,1731736285630; forceNewPlan=false, retain=false 2024-11-16T05:51:29,402 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ff61a3918e3c512081ba7b9c1e1a4948, ASSIGN 2024-11-16T05:51:29,403 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ff61a3918e3c512081ba7b9c1e1a4948, ASSIGN; state=OFFLINE, location=3456ee6a3164,43415,1731736285705; forceNewPlan=false, retain=false 2024-11-16T05:51:29,405 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=883c672d1e7de6e08e15c0c42278e66d, ASSIGN 2024-11-16T05:51:29,406 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=883c672d1e7de6e08e15c0c42278e66d, ASSIGN; state=OFFLINE, location=3456ee6a3164,44781,1731736285630; forceNewPlan=false, retain=false 2024-11-16T05:51:29,408 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3e30199c6045096f8f565c93af9b6088, ASSIGN 2024-11-16T05:51:29,409 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3e30199c6045096f8f565c93af9b6088, ASSIGN; state=OFFLINE, location=3456ee6a3164,33007,1731736285465; forceNewPlan=false, retain=false 2024-11-16T05:51:29,410 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=53faee5a89035a5faf9a235d93b486c9, ASSIGN 2024-11-16T05:51:29,412 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=53faee5a89035a5faf9a235d93b486c9, ASSIGN; state=OFFLINE, location=3456ee6a3164,43415,1731736285705; forceNewPlan=false, retain=false 2024-11-16T05:51:29,413 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=42e678de61063128349f238929e3d296, ASSIGN 2024-11-16T05:51:29,414 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=42e678de61063128349f238929e3d296, ASSIGN; state=OFFLINE, location=3456ee6a3164,44781,1731736285630; forceNewPlan=false, retain=false 2024-11-16T05:51:29,416 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=de75e055385b7b1b204f0aa9c1924d0e, ASSIGN 2024-11-16T05:51:29,418 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=de75e055385b7b1b204f0aa9c1924d0e, ASSIGN; state=OFFLINE, location=3456ee6a3164,33007,1731736285465; forceNewPlan=false, retain=false 2024-11-16T05:51:29,419 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=54bd8d95e5df76bdd2472b3a67cd2b8b, ASSIGN 2024-11-16T05:51:29,421 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=54bd8d95e5df76bdd2472b3a67cd2b8b, ASSIGN; state=OFFLINE, location=3456ee6a3164,43415,1731736285705; forceNewPlan=false, retain=false 2024-11-16T05:51:29,422 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b59301af2f6042086ee9455ab281e7e6, ASSIGN 2024-11-16T05:51:29,424 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b59301af2f6042086ee9455ab281e7e6, ASSIGN; state=OFFLINE, location=3456ee6a3164,33007,1731736285465; forceNewPlan=false, retain=false 2024-11-16T05:51:29,490 INFO [3456ee6a3164:44685 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-16T05:51:29,492 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=ac83f7461cc15e741d0440c6ee2781db, regionState=OPENING, regionLocation=3456ee6a3164,44781,1731736285630 2024-11-16T05:51:29,493 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=10a3e438f5882d261cd799e4d6b75095, regionState=OPENING, regionLocation=3456ee6a3164,44781,1731736285630 2024-11-16T05:51:29,495 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=a6c8ea5676b855dac5a939ace6afadb0, regionState=OPENING, regionLocation=3456ee6a3164,44781,1731736285630 2024-11-16T05:51:29,496 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=ec037e054803eda72bad2d81c6143b00, regionState=OPENING, regionLocation=3456ee6a3164,44781,1731736285630 2024-11-16T05:51:29,497 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=bd79de8195c0452a334d7da789e8f23b, regionState=OPENING, regionLocation=3456ee6a3164,44781,1731736285630 2024-11-16T05:51:29,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ac83f7461cc15e741d0440c6ee2781db, ASSIGN because future has completed 2024-11-16T05:51:29,498 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=c81e1c1af980b616d932866cdd4ff915, regionState=OPENING, regionLocation=3456ee6a3164,44781,1731736285630 2024-11-16T05:51:29,498 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure ac83f7461cc15e741d0440c6ee2781db, server=3456ee6a3164,44781,1731736285630}] 2024-11-16T05:51:29,500 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=10a3e438f5882d261cd799e4d6b75095, ASSIGN because future has completed 2024-11-16T05:51:29,501 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a6c8ea5676b855dac5a939ace6afadb0, ASSIGN because future has completed 2024-11-16T05:51:29,504 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ec037e054803eda72bad2d81c6143b00, ASSIGN because future has completed 2024-11-16T05:51:29,506 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bd79de8195c0452a334d7da789e8f23b, ASSIGN because future has completed 2024-11-16T05:51:29,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd79de8195c0452a334d7da789e8f23b, server=3456ee6a3164,44781,1731736285630}] 2024-11-16T05:51:29,508 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c81e1c1af980b616d932866cdd4ff915, ASSIGN because future has completed 2024-11-16T05:51:29,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure c81e1c1af980b616d932866cdd4ff915, server=3456ee6a3164,44781,1731736285630}] 2024-11-16T05:51:29,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure ec037e054803eda72bad2d81c6143b00, server=3456ee6a3164,44781,1731736285630}] 2024-11-16T05:51:29,528 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure a6c8ea5676b855dac5a939ace6afadb0, server=3456ee6a3164,44781,1731736285630}] 2024-11-16T05:51:29,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10a3e438f5882d261cd799e4d6b75095, server=3456ee6a3164,44781,1731736285630}] 2024-11-16T05:51:29,541 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=883c672d1e7de6e08e15c0c42278e66d, regionState=OPENING, regionLocation=3456ee6a3164,44781,1731736285630 2024-11-16T05:51:29,542 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=42e678de61063128349f238929e3d296, regionState=OPENING, regionLocation=3456ee6a3164,44781,1731736285630 2024-11-16T05:51:29,543 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=afd4d1a884eb847b35b730a1b7a90e0d, regionState=OPENING, regionLocation=3456ee6a3164,43415,1731736285705 2024-11-16T05:51:29,544 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=2beaf8caee4e552158f6bb091a0437bb, regionState=OPENING, regionLocation=3456ee6a3164,43415,1731736285705 2024-11-16T05:51:29,544 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=883c672d1e7de6e08e15c0c42278e66d, ASSIGN because future has completed 2024-11-16T05:51:29,545 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=df4b7c945ebb852a560a10e20c40c386, regionState=OPENING, regionLocation=3456ee6a3164,43415,1731736285705 2024-11-16T05:51:29,546 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=07f1b4025bc2e2627b7dc3f7c79afd8e, regionState=OPENING, regionLocation=3456ee6a3164,43415,1731736285705 2024-11-16T05:51:29,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 883c672d1e7de6e08e15c0c42278e66d, server=3456ee6a3164,44781,1731736285630}] 2024-11-16T05:51:29,548 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=42e678de61063128349f238929e3d296, ASSIGN because future has completed 2024-11-16T05:51:29,550 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=afd4d1a884eb847b35b730a1b7a90e0d, ASSIGN because future has completed 2024-11-16T05:51:29,552 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2beaf8caee4e552158f6bb091a0437bb, ASSIGN because future has completed 2024-11-16T05:51:29,555 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=df4b7c945ebb852a560a10e20c40c386, ASSIGN because future has completed 2024-11-16T05:51:29,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure df4b7c945ebb852a560a10e20c40c386, server=3456ee6a3164,43415,1731736285705}] 2024-11-16T05:51:29,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=07f1b4025bc2e2627b7dc3f7c79afd8e, ASSIGN because future has completed 2024-11-16T05:51:29,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07f1b4025bc2e2627b7dc3f7c79afd8e, server=3456ee6a3164,43415,1731736285705}] 2024-11-16T05:51:29,574 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2beaf8caee4e552158f6bb091a0437bb, server=3456ee6a3164,43415,1731736285705}] 2024-11-16T05:51:29,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure afd4d1a884eb847b35b730a1b7a90e0d, server=3456ee6a3164,43415,1731736285705}] 2024-11-16T05:51:29,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42e678de61063128349f238929e3d296, server=3456ee6a3164,44781,1731736285630}] 2024-11-16T05:51:29,586 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=408fc7812ce9cee6ea99428f155ce5a8, regionState=OPENING, regionLocation=3456ee6a3164,43415,1731736285705 2024-11-16T05:51:29,587 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=5baf9d0bdd8c65c757ad796f93bef122, regionState=OPENING, regionLocation=3456ee6a3164,43415,1731736285705 2024-11-16T05:51:29,587 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=ff61a3918e3c512081ba7b9c1e1a4948, regionState=OPENING, regionLocation=3456ee6a3164,43415,1731736285705 2024-11-16T05:51:29,588 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=53faee5a89035a5faf9a235d93b486c9, regionState=OPENING, regionLocation=3456ee6a3164,43415,1731736285705 2024-11-16T05:51:29,588 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=54bd8d95e5df76bdd2472b3a67cd2b8b, regionState=OPENING, regionLocation=3456ee6a3164,43415,1731736285705 2024-11-16T05:51:29,589 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=ca14297d921e6ca3fee0822ad2bb732e, regionState=OPENING, regionLocation=3456ee6a3164,33007,1731736285465 2024-11-16T05:51:29,589 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=408fc7812ce9cee6ea99428f155ce5a8, ASSIGN because future has completed 2024-11-16T05:51:29,589 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=eb73dbcec883aba378443bd58f1eb724, regionState=OPENING, regionLocation=3456ee6a3164,33007,1731736285465 2024-11-16T05:51:29,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 408fc7812ce9cee6ea99428f155ce5a8, server=3456ee6a3164,43415,1731736285705}] 2024-11-16T05:51:29,591 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5baf9d0bdd8c65c757ad796f93bef122, ASSIGN because future has completed 2024-11-16T05:51:29,592 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ff61a3918e3c512081ba7b9c1e1a4948, ASSIGN because future has completed 2024-11-16T05:51:29,594 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=53faee5a89035a5faf9a235d93b486c9, ASSIGN because future has completed 2024-11-16T05:51:29,595 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 53faee5a89035a5faf9a235d93b486c9, server=3456ee6a3164,43415,1731736285705}] 2024-11-16T05:51:29,595 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=54bd8d95e5df76bdd2472b3a67cd2b8b, ASSIGN because future has completed 2024-11-16T05:51:29,597 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ca14297d921e6ca3fee0822ad2bb732e, ASSIGN because future has completed 2024-11-16T05:51:29,598 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=eb73dbcec883aba378443bd58f1eb724, ASSIGN because future has completed 2024-11-16T05:51:29,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb73dbcec883aba378443bd58f1eb724, server=3456ee6a3164,33007,1731736285465}] 2024-11-16T05:51:29,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure ca14297d921e6ca3fee0822ad2bb732e, server=3456ee6a3164,33007,1731736285465}] 2024-11-16T05:51:29,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 54bd8d95e5df76bdd2472b3a67cd2b8b, server=3456ee6a3164,43415,1731736285705}] 2024-11-16T05:51:29,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff61a3918e3c512081ba7b9c1e1a4948, server=3456ee6a3164,43415,1731736285705}] 2024-11-16T05:51:29,616 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5baf9d0bdd8c65c757ad796f93bef122, server=3456ee6a3164,43415,1731736285705}] 2024-11-16T05:51:29,619 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=9df9b8a2b43462b728e2352478a97347, regionState=OPENING, regionLocation=3456ee6a3164,33007,1731736285465 2024-11-16T05:51:29,620 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=0166203610c1853876df2910d136e997, regionState=OPENING, regionLocation=3456ee6a3164,33007,1731736285465 2024-11-16T05:51:29,620 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=12c6ab39e16284ff5b8ffeb36e672416, regionState=OPENING, regionLocation=3456ee6a3164,33007,1731736285465 2024-11-16T05:51:29,621 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=63baab6432268e686bc2f01a98a6c75f, regionState=OPENING, regionLocation=3456ee6a3164,33007,1731736285465 2024-11-16T05:51:29,621 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=3e30199c6045096f8f565c93af9b6088, regionState=OPENING, regionLocation=3456ee6a3164,33007,1731736285465 2024-11-16T05:51:29,622 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=de75e055385b7b1b204f0aa9c1924d0e, regionState=OPENING, regionLocation=3456ee6a3164,33007,1731736285465 2024-11-16T05:51:29,623 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b59301af2f6042086ee9455ab281e7e6, regionState=OPENING, regionLocation=3456ee6a3164,33007,1731736285465 2024-11-16T05:51:29,623 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9df9b8a2b43462b728e2352478a97347, ASSIGN because future has completed 2024-11-16T05:51:29,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9df9b8a2b43462b728e2352478a97347, server=3456ee6a3164,33007,1731736285465}] 2024-11-16T05:51:29,624 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0166203610c1853876df2910d136e997, ASSIGN because future has completed 2024-11-16T05:51:29,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=12c6ab39e16284ff5b8ffeb36e672416, ASSIGN because future has completed 2024-11-16T05:51:29,627 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=63baab6432268e686bc2f01a98a6c75f, ASSIGN because future has completed 2024-11-16T05:51:29,628 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 63baab6432268e686bc2f01a98a6c75f, server=3456ee6a3164,33007,1731736285465}] 2024-11-16T05:51:29,628 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3e30199c6045096f8f565c93af9b6088, ASSIGN because future has completed 2024-11-16T05:51:29,630 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=de75e055385b7b1b204f0aa9c1924d0e, ASSIGN because future has completed 2024-11-16T05:51:29,631 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b59301af2f6042086ee9455ab281e7e6, ASSIGN because future has completed 2024-11-16T05:51:29,632 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b59301af2f6042086ee9455ab281e7e6, server=3456ee6a3164,33007,1731736285465}] 2024-11-16T05:51:29,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure de75e055385b7b1b204f0aa9c1924d0e, server=3456ee6a3164,33007,1731736285465}] 2024-11-16T05:51:29,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3e30199c6045096f8f565c93af9b6088, server=3456ee6a3164,33007,1731736285465}] 2024-11-16T05:51:29,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12c6ab39e16284ff5b8ffeb36e672416, server=3456ee6a3164,33007,1731736285465}] 2024-11-16T05:51:29,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0166203610c1853876df2910d136e997, server=3456ee6a3164,33007,1731736285465}] 2024-11-16T05:51:29,659 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095. 2024-11-16T05:51:29,660 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 10a3e438f5882d261cd799e4d6b75095, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-16T05:51:29,660 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 10a3e438f5882d261cd799e4d6b75095 2024-11-16T05:51:29,660 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:29,660 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 10a3e438f5882d261cd799e4d6b75095 2024-11-16T05:51:29,660 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 10a3e438f5882d261cd799e4d6b75095 2024-11-16T05:51:29,663 INFO [StoreOpener-10a3e438f5882d261cd799e4d6b75095-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 10a3e438f5882d261cd799e4d6b75095 2024-11-16T05:51:29,665 INFO [StoreOpener-10a3e438f5882d261cd799e4d6b75095-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10a3e438f5882d261cd799e4d6b75095 columnFamilyName colfamily11 2024-11-16T05:51:29,665 DEBUG [StoreOpener-10a3e438f5882d261cd799e4d6b75095-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,666 INFO [StoreOpener-10a3e438f5882d261cd799e4d6b75095-1 {}] regionserver.HStore(327): Store=10a3e438f5882d261cd799e4d6b75095/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,667 INFO [StoreOpener-10a3e438f5882d261cd799e4d6b75095-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 10a3e438f5882d261cd799e4d6b75095 2024-11-16T05:51:29,669 INFO [StoreOpener-10a3e438f5882d261cd799e4d6b75095-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10a3e438f5882d261cd799e4d6b75095 columnFamilyName colfamily21 2024-11-16T05:51:29,669 DEBUG [StoreOpener-10a3e438f5882d261cd799e4d6b75095-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,670 INFO [StoreOpener-10a3e438f5882d261cd799e4d6b75095-1 {}] regionserver.HStore(327): Store=10a3e438f5882d261cd799e4d6b75095/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,670 INFO [StoreOpener-10a3e438f5882d261cd799e4d6b75095-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 10a3e438f5882d261cd799e4d6b75095 2024-11-16T05:51:29,673 INFO [StoreOpener-10a3e438f5882d261cd799e4d6b75095-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10a3e438f5882d261cd799e4d6b75095 columnFamilyName colfamily31 2024-11-16T05:51:29,673 DEBUG [StoreOpener-10a3e438f5882d261cd799e4d6b75095-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,674 INFO [StoreOpener-10a3e438f5882d261cd799e4d6b75095-1 {}] regionserver.HStore(327): Store=10a3e438f5882d261cd799e4d6b75095/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,674 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 10a3e438f5882d261cd799e4d6b75095 2024-11-16T05:51:29,675 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10a3e438f5882d261cd799e4d6b75095 2024-11-16T05:51:29,676 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10a3e438f5882d261cd799e4d6b75095 2024-11-16T05:51:29,677 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 10a3e438f5882d261cd799e4d6b75095 2024-11-16T05:51:29,677 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 10a3e438f5882d261cd799e4d6b75095 2024-11-16T05:51:29,678 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:51:29,680 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 10a3e438f5882d261cd799e4d6b75095 2024-11-16T05:51:29,683 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10a3e438f5882d261cd799e4d6b75095/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:51:29,683 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 10a3e438f5882d261cd799e4d6b75095; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73410887, jitterRate=0.09390746057033539}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:51:29,683 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 10a3e438f5882d261cd799e4d6b75095 2024-11-16T05:51:29,684 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 10a3e438f5882d261cd799e4d6b75095: Running coprocessor pre-open hook at 1731736289661Writing region info on filesystem at 1731736289661Initializing all the Stores at 1731736289662 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289662Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289663 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289663Cleaning up temporary data from old regions at 1731736289677 (+14 ms)Running coprocessor post-open hooks at 1731736289683 (+6 ms)Region opened successfully at 1731736289684 (+1 ms) 2024-11-16T05:51:29,686 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095., pid=36, masterSystemTime=1731736289653 2024-11-16T05:51:29,689 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095. 2024-11-16T05:51:29,689 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095. 2024-11-16T05:51:29,690 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296. 2024-11-16T05:51:29,690 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 42e678de61063128349f238929e3d296, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-16T05:51:29,690 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=10a3e438f5882d261cd799e4d6b75095, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,44781,1731736285630 2024-11-16T05:51:29,690 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 42e678de61063128349f238929e3d296 2024-11-16T05:51:29,690 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:29,691 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 42e678de61063128349f238929e3d296 2024-11-16T05:51:29,691 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 42e678de61063128349f238929e3d296 2024-11-16T05:51:29,693 INFO [StoreOpener-42e678de61063128349f238929e3d296-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 42e678de61063128349f238929e3d296 2024-11-16T05:51:29,694 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10a3e438f5882d261cd799e4d6b75095, server=3456ee6a3164,44781,1731736285630 because future has completed 2024-11-16T05:51:29,696 INFO [StoreOpener-42e678de61063128349f238929e3d296-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42e678de61063128349f238929e3d296 columnFamilyName colfamily11 2024-11-16T05:51:29,697 DEBUG [StoreOpener-42e678de61063128349f238929e3d296-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,698 INFO [StoreOpener-42e678de61063128349f238929e3d296-1 {}] regionserver.HStore(327): Store=42e678de61063128349f238929e3d296/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,698 INFO [StoreOpener-42e678de61063128349f238929e3d296-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 42e678de61063128349f238929e3d296 2024-11-16T05:51:29,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=26 2024-11-16T05:51:29,699 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 10a3e438f5882d261cd799e4d6b75095, server=3456ee6a3164,44781,1731736285630 in 160 msec 2024-11-16T05:51:29,700 INFO [StoreOpener-42e678de61063128349f238929e3d296-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42e678de61063128349f238929e3d296 columnFamilyName colfamily21 2024-11-16T05:51:29,700 DEBUG [StoreOpener-42e678de61063128349f238929e3d296-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,701 INFO [StoreOpener-42e678de61063128349f238929e3d296-1 {}] regionserver.HStore(327): Store=42e678de61063128349f238929e3d296/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,701 INFO [StoreOpener-42e678de61063128349f238929e3d296-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 42e678de61063128349f238929e3d296 2024-11-16T05:51:29,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=10a3e438f5882d261cd799e4d6b75095, ASSIGN in 383 msec 2024-11-16T05:51:29,703 INFO [StoreOpener-42e678de61063128349f238929e3d296-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42e678de61063128349f238929e3d296 columnFamilyName colfamily31 2024-11-16T05:51:29,703 DEBUG [StoreOpener-42e678de61063128349f238929e3d296-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,703 INFO [StoreOpener-42e678de61063128349f238929e3d296-1 {}] regionserver.HStore(327): Store=42e678de61063128349f238929e3d296/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,703 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 42e678de61063128349f238929e3d296 2024-11-16T05:51:29,704 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/42e678de61063128349f238929e3d296 2024-11-16T05:51:29,705 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/42e678de61063128349f238929e3d296 2024-11-16T05:51:29,707 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 42e678de61063128349f238929e3d296 2024-11-16T05:51:29,707 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 42e678de61063128349f238929e3d296 2024-11-16T05:51:29,708 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:51:29,709 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 42e678de61063128349f238929e3d296 2024-11-16T05:51:29,711 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T05:51:29,713 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/42e678de61063128349f238929e3d296/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:51:29,713 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50179, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T05:51:29,714 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 42e678de61063128349f238929e3d296; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59950426, jitterRate=-0.10666903853416443}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:51:29,714 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 42e678de61063128349f238929e3d296 2024-11-16T05:51:29,714 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 42e678de61063128349f238929e3d296: Running coprocessor pre-open hook at 1731736289691Writing region info on filesystem at 1731736289691Initializing all the Stores at 1731736289693 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289693Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289693Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289693Cleaning up temporary data from old regions at 1731736289707 (+14 ms)Running coprocessor post-open hooks at 1731736289714 (+7 ms)Region opened successfully at 1731736289714 2024-11-16T05:51:29,715 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296., pid=42, masterSystemTime=1731736289653 2024-11-16T05:51:29,718 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296. 2024-11-16T05:51:29,719 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296. 2024-11-16T05:51:29,719 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db. 2024-11-16T05:51:29,719 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => ac83f7461cc15e741d0440c6ee2781db, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-16T05:51:29,719 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731736288173.07f1b4025bc2e2627b7dc3f7c79afd8e. 2024-11-16T05:51:29,719 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 07f1b4025bc2e2627b7dc3f7c79afd8e, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731736288173.07f1b4025bc2e2627b7dc3f7c79afd8e.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-16T05:51:29,719 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=42e678de61063128349f238929e3d296, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,44781,1731736285630 2024-11-16T05:51:29,720 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true ac83f7461cc15e741d0440c6ee2781db 2024-11-16T05:51:29,720 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 07f1b4025bc2e2627b7dc3f7c79afd8e 2024-11-16T05:51:29,720 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:29,720 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731736288173.07f1b4025bc2e2627b7dc3f7c79afd8e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:29,720 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 07f1b4025bc2e2627b7dc3f7c79afd8e 2024-11-16T05:51:29,720 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for ac83f7461cc15e741d0440c6ee2781db 2024-11-16T05:51:29,720 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 07f1b4025bc2e2627b7dc3f7c79afd8e 2024-11-16T05:51:29,720 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for ac83f7461cc15e741d0440c6ee2781db 2024-11-16T05:51:29,723 INFO [StoreOpener-07f1b4025bc2e2627b7dc3f7c79afd8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 07f1b4025bc2e2627b7dc3f7c79afd8e 2024-11-16T05:51:29,723 INFO [StoreOpener-ac83f7461cc15e741d0440c6ee2781db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region ac83f7461cc15e741d0440c6ee2781db 2024-11-16T05:51:29,723 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42e678de61063128349f238929e3d296, server=3456ee6a3164,44781,1731736285630 because future has completed 2024-11-16T05:51:29,725 INFO [StoreOpener-07f1b4025bc2e2627b7dc3f7c79afd8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07f1b4025bc2e2627b7dc3f7c79afd8e columnFamilyName colfamily11 2024-11-16T05:51:29,725 DEBUG [StoreOpener-07f1b4025bc2e2627b7dc3f7c79afd8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,725 INFO [StoreOpener-ac83f7461cc15e741d0440c6ee2781db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac83f7461cc15e741d0440c6ee2781db columnFamilyName colfamily11 2024-11-16T05:51:29,726 DEBUG [StoreOpener-ac83f7461cc15e741d0440c6ee2781db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,726 INFO [StoreOpener-07f1b4025bc2e2627b7dc3f7c79afd8e-1 {}] regionserver.HStore(327): Store=07f1b4025bc2e2627b7dc3f7c79afd8e/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,726 INFO [StoreOpener-07f1b4025bc2e2627b7dc3f7c79afd8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 07f1b4025bc2e2627b7dc3f7c79afd8e 2024-11-16T05:51:29,727 INFO [StoreOpener-ac83f7461cc15e741d0440c6ee2781db-1 {}] regionserver.HStore(327): Store=ac83f7461cc15e741d0440c6ee2781db/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,727 INFO [StoreOpener-ac83f7461cc15e741d0440c6ee2781db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region ac83f7461cc15e741d0440c6ee2781db 2024-11-16T05:51:29,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=8 2024-11-16T05:51:29,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 42e678de61063128349f238929e3d296, server=3456ee6a3164,44781,1731736285630 in 142 msec 2024-11-16T05:51:29,729 INFO [StoreOpener-07f1b4025bc2e2627b7dc3f7c79afd8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07f1b4025bc2e2627b7dc3f7c79afd8e columnFamilyName colfamily21 2024-11-16T05:51:29,729 INFO [StoreOpener-ac83f7461cc15e741d0440c6ee2781db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac83f7461cc15e741d0440c6ee2781db columnFamilyName colfamily21 2024-11-16T05:51:29,729 DEBUG [StoreOpener-07f1b4025bc2e2627b7dc3f7c79afd8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,729 DEBUG [StoreOpener-ac83f7461cc15e741d0440c6ee2781db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,730 INFO [StoreOpener-07f1b4025bc2e2627b7dc3f7c79afd8e-1 {}] regionserver.HStore(327): Store=07f1b4025bc2e2627b7dc3f7c79afd8e/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,730 INFO [StoreOpener-07f1b4025bc2e2627b7dc3f7c79afd8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 07f1b4025bc2e2627b7dc3f7c79afd8e 2024-11-16T05:51:29,731 INFO [StoreOpener-ac83f7461cc15e741d0440c6ee2781db-1 {}] regionserver.HStore(327): Store=ac83f7461cc15e741d0440c6ee2781db/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,731 INFO [StoreOpener-ac83f7461cc15e741d0440c6ee2781db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region ac83f7461cc15e741d0440c6ee2781db 2024-11-16T05:51:29,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=42e678de61063128349f238929e3d296, ASSIGN in 413 msec 2024-11-16T05:51:29,733 INFO [StoreOpener-07f1b4025bc2e2627b7dc3f7c79afd8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 07f1b4025bc2e2627b7dc3f7c79afd8e columnFamilyName colfamily31 2024-11-16T05:51:29,733 DEBUG [StoreOpener-07f1b4025bc2e2627b7dc3f7c79afd8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,733 INFO [StoreOpener-ac83f7461cc15e741d0440c6ee2781db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac83f7461cc15e741d0440c6ee2781db columnFamilyName colfamily31 2024-11-16T05:51:29,733 DEBUG [StoreOpener-ac83f7461cc15e741d0440c6ee2781db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,734 INFO [StoreOpener-07f1b4025bc2e2627b7dc3f7c79afd8e-1 {}] regionserver.HStore(327): Store=07f1b4025bc2e2627b7dc3f7c79afd8e/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,734 INFO [StoreOpener-ac83f7461cc15e741d0440c6ee2781db-1 {}] regionserver.HStore(327): Store=ac83f7461cc15e741d0440c6ee2781db/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,734 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 07f1b4025bc2e2627b7dc3f7c79afd8e 2024-11-16T05:51:29,734 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for ac83f7461cc15e741d0440c6ee2781db 2024-11-16T05:51:29,735 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/07f1b4025bc2e2627b7dc3f7c79afd8e 2024-11-16T05:51:29,735 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac83f7461cc15e741d0440c6ee2781db 2024-11-16T05:51:29,736 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/07f1b4025bc2e2627b7dc3f7c79afd8e 2024-11-16T05:51:29,736 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac83f7461cc15e741d0440c6ee2781db 2024-11-16T05:51:29,737 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 07f1b4025bc2e2627b7dc3f7c79afd8e 2024-11-16T05:51:29,737 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 07f1b4025bc2e2627b7dc3f7c79afd8e 2024-11-16T05:51:29,737 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for ac83f7461cc15e741d0440c6ee2781db 2024-11-16T05:51:29,737 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for ac83f7461cc15e741d0440c6ee2781db 2024-11-16T05:51:29,738 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:51:29,738 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:51:29,740 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for ac83f7461cc15e741d0440c6ee2781db 2024-11-16T05:51:29,740 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 07f1b4025bc2e2627b7dc3f7c79afd8e 2024-11-16T05:51:29,743 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/07f1b4025bc2e2627b7dc3f7c79afd8e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:51:29,743 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac83f7461cc15e741d0440c6ee2781db/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:51:29,744 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened ac83f7461cc15e741d0440c6ee2781db; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73961124, jitterRate=0.10210663080215454}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:51:29,744 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 07f1b4025bc2e2627b7dc3f7c79afd8e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74837498, jitterRate=0.11516562104225159}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:51:29,744 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ac83f7461cc15e741d0440c6ee2781db 2024-11-16T05:51:29,744 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 07f1b4025bc2e2627b7dc3f7c79afd8e 2024-11-16T05:51:29,744 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for ac83f7461cc15e741d0440c6ee2781db: Running coprocessor pre-open hook at 1731736289720Writing region info on filesystem at 1731736289720Initializing all the Stores at 1731736289722 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289722Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289722Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289722Cleaning up temporary data from old regions at 1731736289738 (+16 ms)Running coprocessor post-open hooks at 1731736289744 (+6 ms)Region opened successfully at 1731736289744 2024-11-16T05:51:29,744 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 07f1b4025bc2e2627b7dc3f7c79afd8e: Running coprocessor pre-open hook at 1731736289720Writing region info on filesystem at 1731736289720Initializing all the Stores at 1731736289722 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289722Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289722Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289722Cleaning up temporary data from old regions at 1731736289737 (+15 ms)Running coprocessor post-open hooks at 1731736289744 (+7 ms)Region opened successfully at 1731736289744 2024-11-16T05:51:29,746 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db., pid=31, masterSystemTime=1731736289653 2024-11-16T05:51:29,746 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731736288173.07f1b4025bc2e2627b7dc3f7c79afd8e., pid=39, masterSystemTime=1731736289710 2024-11-16T05:51:29,748 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db. 2024-11-16T05:51:29,749 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db. 2024-11-16T05:51:29,749 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915. 2024-11-16T05:51:29,749 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => c81e1c1af980b616d932866cdd4ff915, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-16T05:51:29,750 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=ac83f7461cc15e741d0440c6ee2781db, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,44781,1731736285630 2024-11-16T05:51:29,750 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true c81e1c1af980b616d932866cdd4ff915 2024-11-16T05:51:29,750 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:29,750 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for c81e1c1af980b616d932866cdd4ff915 2024-11-16T05:51:29,750 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731736288173.07f1b4025bc2e2627b7dc3f7c79afd8e. 2024-11-16T05:51:29,750 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for c81e1c1af980b616d932866cdd4ff915 2024-11-16T05:51:29,750 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731736288173.07f1b4025bc2e2627b7dc3f7c79afd8e. 2024-11-16T05:51:29,751 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731736288173.df4b7c945ebb852a560a10e20c40c386. 2024-11-16T05:51:29,751 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => df4b7c945ebb852a560a10e20c40c386, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731736288173.df4b7c945ebb852a560a10e20c40c386.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-16T05:51:29,751 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true df4b7c945ebb852a560a10e20c40c386 2024-11-16T05:51:29,751 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731736288173.df4b7c945ebb852a560a10e20c40c386.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:29,752 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for df4b7c945ebb852a560a10e20c40c386 2024-11-16T05:51:29,752 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for df4b7c945ebb852a560a10e20c40c386 2024-11-16T05:51:29,753 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T05:51:29,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure ac83f7461cc15e741d0440c6ee2781db, server=3456ee6a3164,44781,1731736285630 because future has completed 2024-11-16T05:51:29,754 INFO [StoreOpener-c81e1c1af980b616d932866cdd4ff915-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region c81e1c1af980b616d932866cdd4ff915 2024-11-16T05:51:29,755 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=07f1b4025bc2e2627b7dc3f7c79afd8e, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,43415,1731736285705 2024-11-16T05:51:29,756 INFO [StoreOpener-c81e1c1af980b616d932866cdd4ff915-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c81e1c1af980b616d932866cdd4ff915 columnFamilyName colfamily11 2024-11-16T05:51:29,757 DEBUG [StoreOpener-c81e1c1af980b616d932866cdd4ff915-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,757 INFO [StoreOpener-df4b7c945ebb852a560a10e20c40c386-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region df4b7c945ebb852a560a10e20c40c386 2024-11-16T05:51:29,760 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50229, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T05:51:29,760 INFO [StoreOpener-c81e1c1af980b616d932866cdd4ff915-1 {}] regionserver.HStore(327): Store=c81e1c1af980b616d932866cdd4ff915/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,760 INFO [StoreOpener-c81e1c1af980b616d932866cdd4ff915-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region c81e1c1af980b616d932866cdd4ff915 2024-11-16T05:51:29,761 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 07f1b4025bc2e2627b7dc3f7c79afd8e, server=3456ee6a3164,43415,1731736285705 because future has completed 2024-11-16T05:51:29,761 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44685 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3456ee6a3164,43415,1731736285705, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=07f1b4025bc2e2627b7dc3f7c79afd8e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-16T05:51:29,763 INFO [StoreOpener-df4b7c945ebb852a560a10e20c40c386-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df4b7c945ebb852a560a10e20c40c386 columnFamilyName colfamily11 2024-11-16T05:51:29,763 DEBUG [StoreOpener-df4b7c945ebb852a560a10e20c40c386-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,764 INFO [StoreOpener-c81e1c1af980b616d932866cdd4ff915-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c81e1c1af980b616d932866cdd4ff915 columnFamilyName colfamily21 2024-11-16T05:51:29,764 DEBUG [StoreOpener-c81e1c1af980b616d932866cdd4ff915-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,764 INFO [StoreOpener-df4b7c945ebb852a560a10e20c40c386-1 {}] regionserver.HStore(327): Store=df4b7c945ebb852a560a10e20c40c386/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=28 2024-11-16T05:51:29,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure ac83f7461cc15e741d0440c6ee2781db, server=3456ee6a3164,44781,1731736285630 in 262 msec 2024-11-16T05:51:29,767 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731736288173.0166203610c1853876df2910d136e997. 2024-11-16T05:51:29,768 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 0166203610c1853876df2910d136e997, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731736288173.0166203610c1853876df2910d136e997.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-16T05:51:29,768 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 0166203610c1853876df2910d136e997 2024-11-16T05:51:29,769 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ac83f7461cc15e741d0440c6ee2781db, ASSIGN in 450 msec 2024-11-16T05:51:29,769 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731736288173.0166203610c1853876df2910d136e997.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:29,765 INFO [StoreOpener-df4b7c945ebb852a560a10e20c40c386-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region df4b7c945ebb852a560a10e20c40c386 2024-11-16T05:51:29,769 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 0166203610c1853876df2910d136e997 2024-11-16T05:51:29,769 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 0166203610c1853876df2910d136e997 2024-11-16T05:51:29,770 INFO [StoreOpener-c81e1c1af980b616d932866cdd4ff915-1 {}] regionserver.HStore(327): Store=c81e1c1af980b616d932866cdd4ff915/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,770 INFO [StoreOpener-c81e1c1af980b616d932866cdd4ff915-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region c81e1c1af980b616d932866cdd4ff915 2024-11-16T05:51:29,771 INFO [StoreOpener-df4b7c945ebb852a560a10e20c40c386-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df4b7c945ebb852a560a10e20c40c386 columnFamilyName colfamily21 2024-11-16T05:51:29,771 DEBUG [StoreOpener-df4b7c945ebb852a560a10e20c40c386-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,773 INFO [StoreOpener-df4b7c945ebb852a560a10e20c40c386-1 {}] regionserver.HStore(327): Store=df4b7c945ebb852a560a10e20c40c386/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,773 INFO [StoreOpener-df4b7c945ebb852a560a10e20c40c386-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region df4b7c945ebb852a560a10e20c40c386 2024-11-16T05:51:29,775 INFO [StoreOpener-c81e1c1af980b616d932866cdd4ff915-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c81e1c1af980b616d932866cdd4ff915 columnFamilyName colfamily31 2024-11-16T05:51:29,775 DEBUG [StoreOpener-c81e1c1af980b616d932866cdd4ff915-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,777 INFO [StoreOpener-c81e1c1af980b616d932866cdd4ff915-1 {}] regionserver.HStore(327): Store=c81e1c1af980b616d932866cdd4ff915/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,777 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for c81e1c1af980b616d932866cdd4ff915 2024-11-16T05:51:29,779 INFO [StoreOpener-df4b7c945ebb852a560a10e20c40c386-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df4b7c945ebb852a560a10e20c40c386 columnFamilyName colfamily31 2024-11-16T05:51:29,779 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c81e1c1af980b616d932866cdd4ff915 2024-11-16T05:51:29,779 DEBUG [StoreOpener-df4b7c945ebb852a560a10e20c40c386-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,780 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c81e1c1af980b616d932866cdd4ff915 2024-11-16T05:51:29,781 INFO [StoreOpener-df4b7c945ebb852a560a10e20c40c386-1 {}] regionserver.HStore(327): Store=df4b7c945ebb852a560a10e20c40c386/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,781 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for df4b7c945ebb852a560a10e20c40c386 2024-11-16T05:51:29,782 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/df4b7c945ebb852a560a10e20c40c386 2024-11-16T05:51:29,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=19 2024-11-16T05:51:29,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 07f1b4025bc2e2627b7dc3f7c79afd8e, server=3456ee6a3164,43415,1731736285705 in 206 msec 2024-11-16T05:51:29,783 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for c81e1c1af980b616d932866cdd4ff915 2024-11-16T05:51:29,783 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for c81e1c1af980b616d932866cdd4ff915 2024-11-16T05:51:29,784 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/df4b7c945ebb852a560a10e20c40c386 2024-11-16T05:51:29,784 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:51:29,786 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for df4b7c945ebb852a560a10e20c40c386 2024-11-16T05:51:29,786 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for c81e1c1af980b616d932866cdd4ff915 2024-11-16T05:51:29,786 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for df4b7c945ebb852a560a10e20c40c386 2024-11-16T05:51:29,787 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=07f1b4025bc2e2627b7dc3f7c79afd8e, ASSIGN in 467 msec 2024-11-16T05:51:29,787 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:51:29,789 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for df4b7c945ebb852a560a10e20c40c386 2024-11-16T05:51:29,789 INFO [StoreOpener-0166203610c1853876df2910d136e997-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 0166203610c1853876df2910d136e997 2024-11-16T05:51:29,790 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c81e1c1af980b616d932866cdd4ff915/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:51:29,791 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened c81e1c1af980b616d932866cdd4ff915; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68064068, jitterRate=0.014233648777008057}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:51:29,791 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c81e1c1af980b616d932866cdd4ff915 2024-11-16T05:51:29,791 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for c81e1c1af980b616d932866cdd4ff915: Running coprocessor pre-open hook at 1731736289750Writing region info on filesystem at 1731736289750Initializing all the Stores at 1731736289752 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289753 (+1 ms)Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289753Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289753Cleaning up temporary data from old regions at 1731736289783 (+30 ms)Running coprocessor post-open hooks at 1731736289791 (+8 ms)Region opened successfully at 1731736289791 2024-11-16T05:51:29,792 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915., pid=33, masterSystemTime=1731736289653 2024-11-16T05:51:29,793 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/df4b7c945ebb852a560a10e20c40c386/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:51:29,793 INFO [StoreOpener-0166203610c1853876df2910d136e997-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0166203610c1853876df2910d136e997 columnFamilyName colfamily11 2024-11-16T05:51:29,793 DEBUG [StoreOpener-0166203610c1853876df2910d136e997-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,794 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened df4b7c945ebb852a560a10e20c40c386; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71885911, jitterRate=0.07118354737758636}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:51:29,794 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for df4b7c945ebb852a560a10e20c40c386 2024-11-16T05:51:29,794 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for df4b7c945ebb852a560a10e20c40c386: Running coprocessor pre-open hook at 1731736289752Writing region info on filesystem at 1731736289752Initializing all the Stores at 1731736289753 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289754 (+1 ms)Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289757 (+3 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289757Cleaning up temporary data from old regions at 1731736289786 (+29 ms)Running coprocessor post-open hooks at 1731736289794 (+8 ms)Region opened successfully at 1731736289794 2024-11-16T05:51:29,795 INFO [StoreOpener-0166203610c1853876df2910d136e997-1 {}] regionserver.HStore(327): Store=0166203610c1853876df2910d136e997/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,795 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731736288173.df4b7c945ebb852a560a10e20c40c386., pid=38, masterSystemTime=1731736289710 2024-11-16T05:51:29,796 INFO [StoreOpener-0166203610c1853876df2910d136e997-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 0166203610c1853876df2910d136e997 2024-11-16T05:51:29,796 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915. 2024-11-16T05:51:29,796 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915. 2024-11-16T05:51:29,797 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0. 2024-11-16T05:51:29,797 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => a6c8ea5676b855dac5a939ace6afadb0, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-16T05:51:29,797 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true a6c8ea5676b855dac5a939ace6afadb0 2024-11-16T05:51:29,797 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:29,798 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for a6c8ea5676b855dac5a939ace6afadb0 2024-11-16T05:51:29,798 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for a6c8ea5676b855dac5a939ace6afadb0 2024-11-16T05:51:29,798 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=c81e1c1af980b616d932866cdd4ff915, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,44781,1731736285630 2024-11-16T05:51:29,800 INFO [StoreOpener-0166203610c1853876df2910d136e997-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0166203610c1853876df2910d136e997 columnFamilyName colfamily21 2024-11-16T05:51:29,800 DEBUG [StoreOpener-0166203610c1853876df2910d136e997-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,801 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731736288173.df4b7c945ebb852a560a10e20c40c386. 2024-11-16T05:51:29,801 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731736288173.df4b7c945ebb852a560a10e20c40c386. 2024-11-16T05:51:29,801 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731736288173.5baf9d0bdd8c65c757ad796f93bef122. 2024-11-16T05:51:29,802 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 5baf9d0bdd8c65c757ad796f93bef122, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731736288173.5baf9d0bdd8c65c757ad796f93bef122.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-16T05:51:29,802 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 5baf9d0bdd8c65c757ad796f93bef122 2024-11-16T05:51:29,802 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731736288173.5baf9d0bdd8c65c757ad796f93bef122.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:29,802 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 5baf9d0bdd8c65c757ad796f93bef122 2024-11-16T05:51:29,802 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 5baf9d0bdd8c65c757ad796f93bef122 2024-11-16T05:51:29,803 INFO [StoreOpener-0166203610c1853876df2910d136e997-1 {}] regionserver.HStore(327): Store=0166203610c1853876df2910d136e997/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,803 INFO [StoreOpener-0166203610c1853876df2910d136e997-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 0166203610c1853876df2910d136e997 2024-11-16T05:51:29,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure c81e1c1af980b616d932866cdd4ff915, server=3456ee6a3164,44781,1731736285630 because future has completed 2024-11-16T05:51:29,803 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=df4b7c945ebb852a560a10e20c40c386, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,43415,1731736285705 2024-11-16T05:51:29,805 INFO [StoreOpener-0166203610c1853876df2910d136e997-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0166203610c1853876df2910d136e997 columnFamilyName colfamily31 2024-11-16T05:51:29,805 DEBUG [StoreOpener-0166203610c1853876df2910d136e997-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,806 INFO [StoreOpener-0166203610c1853876df2910d136e997-1 {}] regionserver.HStore(327): Store=0166203610c1853876df2910d136e997/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,807 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 0166203610c1853876df2910d136e997 2024-11-16T05:51:29,807 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure df4b7c945ebb852a560a10e20c40c386, server=3456ee6a3164,43415,1731736285705 because future has completed 2024-11-16T05:51:29,808 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0166203610c1853876df2910d136e997 2024-11-16T05:51:29,809 INFO [StoreOpener-a6c8ea5676b855dac5a939ace6afadb0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region a6c8ea5676b855dac5a939ace6afadb0 2024-11-16T05:51:29,810 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0166203610c1853876df2910d136e997 2024-11-16T05:51:29,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=13 2024-11-16T05:51:29,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure c81e1c1af980b616d932866cdd4ff915, server=3456ee6a3164,44781,1731736285630 in 294 msec 2024-11-16T05:51:29,812 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 0166203610c1853876df2910d136e997 2024-11-16T05:51:29,813 INFO [StoreOpener-a6c8ea5676b855dac5a939ace6afadb0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6c8ea5676b855dac5a939ace6afadb0 columnFamilyName colfamily11 2024-11-16T05:51:29,813 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 0166203610c1853876df2910d136e997 2024-11-16T05:51:29,813 DEBUG [StoreOpener-a6c8ea5676b855dac5a939ace6afadb0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,814 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:51:29,814 INFO [StoreOpener-a6c8ea5676b855dac5a939ace6afadb0-1 {}] regionserver.HStore(327): Store=a6c8ea5676b855dac5a939ace6afadb0/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,814 INFO [StoreOpener-a6c8ea5676b855dac5a939ace6afadb0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region a6c8ea5676b855dac5a939ace6afadb0 2024-11-16T05:51:29,815 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c81e1c1af980b616d932866cdd4ff915, ASSIGN in 495 msec 2024-11-16T05:51:29,816 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 0166203610c1853876df2910d136e997 2024-11-16T05:51:29,816 INFO [StoreOpener-a6c8ea5676b855dac5a939ace6afadb0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6c8ea5676b855dac5a939ace6afadb0 columnFamilyName colfamily21 2024-11-16T05:51:29,817 DEBUG [StoreOpener-a6c8ea5676b855dac5a939ace6afadb0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,818 INFO [StoreOpener-a6c8ea5676b855dac5a939ace6afadb0-1 {}] regionserver.HStore(327): Store=a6c8ea5676b855dac5a939ace6afadb0/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,818 INFO [StoreOpener-a6c8ea5676b855dac5a939ace6afadb0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region a6c8ea5676b855dac5a939ace6afadb0 2024-11-16T05:51:29,820 INFO [StoreOpener-a6c8ea5676b855dac5a939ace6afadb0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6c8ea5676b855dac5a939ace6afadb0 columnFamilyName colfamily31 2024-11-16T05:51:29,820 DEBUG [StoreOpener-a6c8ea5676b855dac5a939ace6afadb0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,821 INFO [StoreOpener-a6c8ea5676b855dac5a939ace6afadb0-1 {}] regionserver.HStore(327): Store=a6c8ea5676b855dac5a939ace6afadb0/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,821 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for a6c8ea5676b855dac5a939ace6afadb0 2024-11-16T05:51:29,821 INFO [StoreOpener-5baf9d0bdd8c65c757ad796f93bef122-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 5baf9d0bdd8c65c757ad796f93bef122 2024-11-16T05:51:29,822 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=22 2024-11-16T05:51:29,822 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a6c8ea5676b855dac5a939ace6afadb0 2024-11-16T05:51:29,822 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure df4b7c945ebb852a560a10e20c40c386, server=3456ee6a3164,43415,1731736285705 in 262 msec 2024-11-16T05:51:29,823 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a6c8ea5676b855dac5a939ace6afadb0 2024-11-16T05:51:29,824 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0166203610c1853876df2910d136e997/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:51:29,824 INFO [StoreOpener-5baf9d0bdd8c65c757ad796f93bef122-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5baf9d0bdd8c65c757ad796f93bef122 columnFamilyName colfamily11 2024-11-16T05:51:29,825 DEBUG [StoreOpener-5baf9d0bdd8c65c757ad796f93bef122-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,825 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 0166203610c1853876df2910d136e997; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71087296, jitterRate=0.05928325653076172}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:51:29,825 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0166203610c1853876df2910d136e997 2024-11-16T05:51:29,825 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 0166203610c1853876df2910d136e997: Running coprocessor pre-open hook at 1731736289778Writing region info on filesystem at 1731736289778Initializing all the Stores at 1731736289782 (+4 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289782Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289785 (+3 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289785Cleaning up temporary data from old regions at 1731736289813 (+28 ms)Running coprocessor post-open hooks at 1731736289825 (+12 ms)Region opened successfully at 1731736289825 2024-11-16T05:51:29,825 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for a6c8ea5676b855dac5a939ace6afadb0 2024-11-16T05:51:29,825 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for a6c8ea5676b855dac5a939ace6afadb0 2024-11-16T05:51:29,825 INFO [StoreOpener-5baf9d0bdd8c65c757ad796f93bef122-1 {}] regionserver.HStore(327): Store=5baf9d0bdd8c65c757ad796f93bef122/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,825 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=df4b7c945ebb852a560a10e20c40c386, ASSIGN in 507 msec 2024-11-16T05:51:29,826 INFO [StoreOpener-5baf9d0bdd8c65c757ad796f93bef122-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 5baf9d0bdd8c65c757ad796f93bef122 2024-11-16T05:51:29,826 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:51:29,827 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731736288173.0166203610c1853876df2910d136e997., pid=56, masterSystemTime=1731736289752 2024-11-16T05:51:29,828 INFO [StoreOpener-5baf9d0bdd8c65c757ad796f93bef122-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5baf9d0bdd8c65c757ad796f93bef122 columnFamilyName colfamily21 2024-11-16T05:51:29,828 DEBUG [StoreOpener-5baf9d0bdd8c65c757ad796f93bef122-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,829 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for a6c8ea5676b855dac5a939ace6afadb0 2024-11-16T05:51:29,829 INFO [StoreOpener-5baf9d0bdd8c65c757ad796f93bef122-1 {}] regionserver.HStore(327): Store=5baf9d0bdd8c65c757ad796f93bef122/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,830 INFO [StoreOpener-5baf9d0bdd8c65c757ad796f93bef122-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 5baf9d0bdd8c65c757ad796f93bef122 2024-11-16T05:51:29,830 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731736288173.0166203610c1853876df2910d136e997. 2024-11-16T05:51:29,831 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731736288173.0166203610c1853876df2910d136e997. 2024-11-16T05:51:29,831 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731736288173.3e30199c6045096f8f565c93af9b6088. 2024-11-16T05:51:29,831 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 3e30199c6045096f8f565c93af9b6088, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731736288173.3e30199c6045096f8f565c93af9b6088.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-16T05:51:29,832 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 3e30199c6045096f8f565c93af9b6088 2024-11-16T05:51:29,832 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=0166203610c1853876df2910d136e997, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,33007,1731736285465 2024-11-16T05:51:29,832 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731736288173.3e30199c6045096f8f565c93af9b6088.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:29,832 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 3e30199c6045096f8f565c93af9b6088 2024-11-16T05:51:29,832 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 3e30199c6045096f8f565c93af9b6088 2024-11-16T05:51:29,834 INFO [StoreOpener-5baf9d0bdd8c65c757ad796f93bef122-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5baf9d0bdd8c65c757ad796f93bef122 columnFamilyName colfamily31 2024-11-16T05:51:29,834 DEBUG [StoreOpener-5baf9d0bdd8c65c757ad796f93bef122-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,835 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a6c8ea5676b855dac5a939ace6afadb0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:51:29,835 INFO [StoreOpener-5baf9d0bdd8c65c757ad796f93bef122-1 {}] regionserver.HStore(327): Store=5baf9d0bdd8c65c757ad796f93bef122/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,835 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 5baf9d0bdd8c65c757ad796f93bef122 2024-11-16T05:51:29,836 INFO [StoreOpener-3e30199c6045096f8f565c93af9b6088-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 3e30199c6045096f8f565c93af9b6088 2024-11-16T05:51:29,837 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5baf9d0bdd8c65c757ad796f93bef122 2024-11-16T05:51:29,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0166203610c1853876df2910d136e997, server=3456ee6a3164,33007,1731736285465 because future has completed 2024-11-16T05:51:29,837 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened a6c8ea5676b855dac5a939ace6afadb0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60075781, jitterRate=-0.10480110347270966}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:51:29,837 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a6c8ea5676b855dac5a939ace6afadb0 2024-11-16T05:51:29,837 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5baf9d0bdd8c65c757ad796f93bef122 2024-11-16T05:51:29,838 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for a6c8ea5676b855dac5a939ace6afadb0: Running coprocessor pre-open hook at 1731736289798Writing region info on filesystem at 1731736289798Initializing all the Stores at 1731736289799 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289799Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289809 (+10 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289809Cleaning up temporary data from old regions at 1731736289825 (+16 ms)Running coprocessor post-open hooks at 1731736289838 (+13 ms)Region opened successfully at 1731736289838 2024-11-16T05:51:29,839 INFO [StoreOpener-3e30199c6045096f8f565c93af9b6088-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3e30199c6045096f8f565c93af9b6088 columnFamilyName colfamily11 2024-11-16T05:51:29,839 DEBUG [StoreOpener-3e30199c6045096f8f565c93af9b6088-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,839 INFO [StoreOpener-3e30199c6045096f8f565c93af9b6088-1 {}] regionserver.HStore(327): Store=3e30199c6045096f8f565c93af9b6088/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,840 INFO [StoreOpener-3e30199c6045096f8f565c93af9b6088-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 3e30199c6045096f8f565c93af9b6088 2024-11-16T05:51:29,840 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0., pid=35, masterSystemTime=1731736289653 2024-11-16T05:51:29,840 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 5baf9d0bdd8c65c757ad796f93bef122 2024-11-16T05:51:29,841 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 5baf9d0bdd8c65c757ad796f93bef122 2024-11-16T05:51:29,842 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:51:29,844 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0. 2024-11-16T05:51:29,844 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0. 2024-11-16T05:51:29,844 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 5baf9d0bdd8c65c757ad796f93bef122 2024-11-16T05:51:29,844 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d. 2024-11-16T05:51:29,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=21 2024-11-16T05:51:29,845 INFO [StoreOpener-3e30199c6045096f8f565c93af9b6088-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3e30199c6045096f8f565c93af9b6088 columnFamilyName colfamily21 2024-11-16T05:51:29,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 0166203610c1853876df2910d136e997, server=3456ee6a3164,33007,1731736285465 in 192 msec 2024-11-16T05:51:29,845 DEBUG [StoreOpener-3e30199c6045096f8f565c93af9b6088-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,845 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 883c672d1e7de6e08e15c0c42278e66d, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-16T05:51:29,845 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 883c672d1e7de6e08e15c0c42278e66d 2024-11-16T05:51:29,845 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:29,846 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 883c672d1e7de6e08e15c0c42278e66d 2024-11-16T05:51:29,846 INFO [StoreOpener-3e30199c6045096f8f565c93af9b6088-1 {}] regionserver.HStore(327): Store=3e30199c6045096f8f565c93af9b6088/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,846 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 883c672d1e7de6e08e15c0c42278e66d 2024-11-16T05:51:29,846 INFO [StoreOpener-3e30199c6045096f8f565c93af9b6088-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 3e30199c6045096f8f565c93af9b6088 2024-11-16T05:51:29,848 INFO [StoreOpener-3e30199c6045096f8f565c93af9b6088-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3e30199c6045096f8f565c93af9b6088 columnFamilyName colfamily31 2024-11-16T05:51:29,849 DEBUG [StoreOpener-3e30199c6045096f8f565c93af9b6088-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,849 INFO [StoreOpener-883c672d1e7de6e08e15c0c42278e66d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 883c672d1e7de6e08e15c0c42278e66d 2024-11-16T05:51:29,849 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5baf9d0bdd8c65c757ad796f93bef122/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:51:29,850 INFO [StoreOpener-3e30199c6045096f8f565c93af9b6088-1 {}] regionserver.HStore(327): Store=3e30199c6045096f8f565c93af9b6088/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,850 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 3e30199c6045096f8f565c93af9b6088 2024-11-16T05:51:29,850 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=0166203610c1853876df2910d136e997, ASSIGN in 529 msec 2024-11-16T05:51:29,850 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 5baf9d0bdd8c65c757ad796f93bef122; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64717826, jitterRate=-0.03562924265861511}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:51:29,850 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5baf9d0bdd8c65c757ad796f93bef122 2024-11-16T05:51:29,850 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 5baf9d0bdd8c65c757ad796f93bef122: Running coprocessor pre-open hook at 1731736289802Writing region info on filesystem at 1731736289802Initializing all the Stores at 1731736289804 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289804Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289821 (+17 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289821Cleaning up temporary data from old regions at 1731736289841 (+20 ms)Running coprocessor post-open hooks at 1731736289850 (+9 ms)Region opened successfully at 1731736289850 2024-11-16T05:51:29,851 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3e30199c6045096f8f565c93af9b6088 2024-11-16T05:51:29,852 INFO [StoreOpener-883c672d1e7de6e08e15c0c42278e66d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 883c672d1e7de6e08e15c0c42278e66d columnFamilyName colfamily11 2024-11-16T05:51:29,852 DEBUG [StoreOpener-883c672d1e7de6e08e15c0c42278e66d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,852 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3e30199c6045096f8f565c93af9b6088 2024-11-16T05:51:29,853 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731736288173.5baf9d0bdd8c65c757ad796f93bef122., pid=49, masterSystemTime=1731736289710 2024-11-16T05:51:29,853 INFO [StoreOpener-883c672d1e7de6e08e15c0c42278e66d-1 {}] regionserver.HStore(327): Store=883c672d1e7de6e08e15c0c42278e66d/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,853 INFO [StoreOpener-883c672d1e7de6e08e15c0c42278e66d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 883c672d1e7de6e08e15c0c42278e66d 2024-11-16T05:51:29,854 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=a6c8ea5676b855dac5a939ace6afadb0, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,44781,1731736285630 2024-11-16T05:51:29,854 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 3e30199c6045096f8f565c93af9b6088 2024-11-16T05:51:29,854 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 3e30199c6045096f8f565c93af9b6088 2024-11-16T05:51:29,855 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:51:29,855 INFO [StoreOpener-883c672d1e7de6e08e15c0c42278e66d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 883c672d1e7de6e08e15c0c42278e66d columnFamilyName colfamily21 2024-11-16T05:51:29,855 DEBUG [StoreOpener-883c672d1e7de6e08e15c0c42278e66d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,856 INFO [StoreOpener-883c672d1e7de6e08e15c0c42278e66d-1 {}] regionserver.HStore(327): Store=883c672d1e7de6e08e15c0c42278e66d/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,856 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731736288173.5baf9d0bdd8c65c757ad796f93bef122. 2024-11-16T05:51:29,856 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731736288173.5baf9d0bdd8c65c757ad796f93bef122. 2024-11-16T05:51:29,856 INFO [StoreOpener-883c672d1e7de6e08e15c0c42278e66d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 883c672d1e7de6e08e15c0c42278e66d 2024-11-16T05:51:29,857 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731736288173.54bd8d95e5df76bdd2472b3a67cd2b8b. 2024-11-16T05:51:29,857 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 3e30199c6045096f8f565c93af9b6088 2024-11-16T05:51:29,857 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 54bd8d95e5df76bdd2472b3a67cd2b8b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731736288173.54bd8d95e5df76bdd2472b3a67cd2b8b.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-16T05:51:29,858 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 54bd8d95e5df76bdd2472b3a67cd2b8b 2024-11-16T05:51:29,858 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731736288173.54bd8d95e5df76bdd2472b3a67cd2b8b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:29,859 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 54bd8d95e5df76bdd2472b3a67cd2b8b 2024-11-16T05:51:29,859 INFO [StoreOpener-883c672d1e7de6e08e15c0c42278e66d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 883c672d1e7de6e08e15c0c42278e66d columnFamilyName colfamily31 2024-11-16T05:51:29,859 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 54bd8d95e5df76bdd2472b3a67cd2b8b 2024-11-16T05:51:29,859 DEBUG [StoreOpener-883c672d1e7de6e08e15c0c42278e66d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure a6c8ea5676b855dac5a939ace6afadb0, server=3456ee6a3164,44781,1731736285630 because future has completed 2024-11-16T05:51:29,860 INFO [StoreOpener-883c672d1e7de6e08e15c0c42278e66d-1 {}] regionserver.HStore(327): Store=883c672d1e7de6e08e15c0c42278e66d/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,860 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 883c672d1e7de6e08e15c0c42278e66d 2024-11-16T05:51:29,862 INFO [StoreOpener-54bd8d95e5df76bdd2472b3a67cd2b8b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 54bd8d95e5df76bdd2472b3a67cd2b8b 2024-11-16T05:51:29,862 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/883c672d1e7de6e08e15c0c42278e66d 2024-11-16T05:51:29,863 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/883c672d1e7de6e08e15c0c42278e66d 2024-11-16T05:51:29,864 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3e30199c6045096f8f565c93af9b6088/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:51:29,865 INFO [StoreOpener-54bd8d95e5df76bdd2472b3a67cd2b8b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54bd8d95e5df76bdd2472b3a67cd2b8b columnFamilyName colfamily11 2024-11-16T05:51:29,865 DEBUG [StoreOpener-54bd8d95e5df76bdd2472b3a67cd2b8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,865 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 883c672d1e7de6e08e15c0c42278e66d 2024-11-16T05:51:29,865 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 883c672d1e7de6e08e15c0c42278e66d 2024-11-16T05:51:29,865 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 3e30199c6045096f8f565c93af9b6088; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60377414, jitterRate=-0.1003064215183258}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:51:29,865 INFO [StoreOpener-54bd8d95e5df76bdd2472b3a67cd2b8b-1 {}] regionserver.HStore(327): Store=54bd8d95e5df76bdd2472b3a67cd2b8b/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,866 INFO [StoreOpener-54bd8d95e5df76bdd2472b3a67cd2b8b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 54bd8d95e5df76bdd2472b3a67cd2b8b 2024-11-16T05:51:29,866 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3e30199c6045096f8f565c93af9b6088 2024-11-16T05:51:29,866 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 3e30199c6045096f8f565c93af9b6088: Running coprocessor pre-open hook at 1731736289833Writing region info on filesystem at 1731736289833Initializing all the Stores at 1731736289835 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289835Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289835Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289835Cleaning up temporary data from old regions at 1731736289854 (+19 ms)Running coprocessor post-open hooks at 1731736289866 (+12 ms)Region opened successfully at 1731736289866 2024-11-16T05:51:29,866 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:51:29,868 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731736288173.3e30199c6045096f8f565c93af9b6088., pid=54, masterSystemTime=1731736289752 2024-11-16T05:51:29,869 INFO [StoreOpener-54bd8d95e5df76bdd2472b3a67cd2b8b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54bd8d95e5df76bdd2472b3a67cd2b8b columnFamilyName colfamily21 2024-11-16T05:51:29,869 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 883c672d1e7de6e08e15c0c42278e66d 2024-11-16T05:51:29,869 DEBUG [StoreOpener-54bd8d95e5df76bdd2472b3a67cd2b8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,870 INFO [StoreOpener-54bd8d95e5df76bdd2472b3a67cd2b8b-1 {}] regionserver.HStore(327): Store=54bd8d95e5df76bdd2472b3a67cd2b8b/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,871 INFO [StoreOpener-54bd8d95e5df76bdd2472b3a67cd2b8b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 54bd8d95e5df76bdd2472b3a67cd2b8b 2024-11-16T05:51:29,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=23 2024-11-16T05:51:29,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure a6c8ea5676b855dac5a939ace6afadb0, server=3456ee6a3164,44781,1731736285630 in 337 msec 2024-11-16T05:51:29,871 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731736288173.3e30199c6045096f8f565c93af9b6088. 2024-11-16T05:51:29,872 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731736288173.3e30199c6045096f8f565c93af9b6088. 2024-11-16T05:51:29,872 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731736288173.ca14297d921e6ca3fee0822ad2bb732e. 2024-11-16T05:51:29,872 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => ca14297d921e6ca3fee0822ad2bb732e, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731736288173.ca14297d921e6ca3fee0822ad2bb732e.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-16T05:51:29,873 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true ca14297d921e6ca3fee0822ad2bb732e 2024-11-16T05:51:29,873 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731736288173.ca14297d921e6ca3fee0822ad2bb732e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:29,873 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for ca14297d921e6ca3fee0822ad2bb732e 2024-11-16T05:51:29,873 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for ca14297d921e6ca3fee0822ad2bb732e 2024-11-16T05:51:29,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a6c8ea5676b855dac5a939ace6afadb0, ASSIGN in 556 msec 2024-11-16T05:51:29,874 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/883c672d1e7de6e08e15c0c42278e66d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:51:29,875 INFO [StoreOpener-54bd8d95e5df76bdd2472b3a67cd2b8b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54bd8d95e5df76bdd2472b3a67cd2b8b columnFamilyName colfamily31 2024-11-16T05:51:29,875 DEBUG [StoreOpener-54bd8d95e5df76bdd2472b3a67cd2b8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,875 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 883c672d1e7de6e08e15c0c42278e66d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75200703, jitterRate=0.12057779729366302}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:51:29,876 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 883c672d1e7de6e08e15c0c42278e66d 2024-11-16T05:51:29,876 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 883c672d1e7de6e08e15c0c42278e66d: Running coprocessor pre-open hook at 1731736289846Writing region info on filesystem at 1731736289846Initializing all the Stores at 1731736289848 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289848Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289850 (+2 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289850Cleaning up temporary data from old regions at 1731736289865 (+15 ms)Running coprocessor post-open hooks at 1731736289876 (+11 ms)Region opened successfully at 1731736289876 2024-11-16T05:51:29,876 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=3e30199c6045096f8f565c93af9b6088, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,33007,1731736285465 2024-11-16T05:51:29,878 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d., pid=37, masterSystemTime=1731736289653 2024-11-16T05:51:29,879 INFO [StoreOpener-54bd8d95e5df76bdd2472b3a67cd2b8b-1 {}] regionserver.HStore(327): Store=54bd8d95e5df76bdd2472b3a67cd2b8b/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,880 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 54bd8d95e5df76bdd2472b3a67cd2b8b 2024-11-16T05:51:29,882 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/54bd8d95e5df76bdd2472b3a67cd2b8b 2024-11-16T05:51:29,884 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/54bd8d95e5df76bdd2472b3a67cd2b8b 2024-11-16T05:51:29,886 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 54bd8d95e5df76bdd2472b3a67cd2b8b 2024-11-16T05:51:29,886 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 54bd8d95e5df76bdd2472b3a67cd2b8b 2024-11-16T05:51:29,886 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:51:29,889 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d. 2024-11-16T05:51:29,889 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d. 2024-11-16T05:51:29,889 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 54bd8d95e5df76bdd2472b3a67cd2b8b 2024-11-16T05:51:29,889 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b. 2024-11-16T05:51:29,890 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => bd79de8195c0452a334d7da789e8f23b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-16T05:51:29,890 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true bd79de8195c0452a334d7da789e8f23b 2024-11-16T05:51:29,890 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:29,890 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for bd79de8195c0452a334d7da789e8f23b 2024-11-16T05:51:29,891 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for bd79de8195c0452a334d7da789e8f23b 2024-11-16T05:51:29,893 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=883c672d1e7de6e08e15c0c42278e66d, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,44781,1731736285630 2024-11-16T05:51:29,896 INFO [StoreOpener-ca14297d921e6ca3fee0822ad2bb732e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region ca14297d921e6ca3fee0822ad2bb732e 2024-11-16T05:51:29,896 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3e30199c6045096f8f565c93af9b6088, server=3456ee6a3164,33007,1731736285465 because future has completed 2024-11-16T05:51:29,898 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/54bd8d95e5df76bdd2472b3a67cd2b8b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:51:29,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 883c672d1e7de6e08e15c0c42278e66d, server=3456ee6a3164,44781,1731736285630 because future has completed 2024-11-16T05:51:29,899 INFO [StoreOpener-ca14297d921e6ca3fee0822ad2bb732e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca14297d921e6ca3fee0822ad2bb732e columnFamilyName colfamily11 2024-11-16T05:51:29,899 DEBUG [StoreOpener-ca14297d921e6ca3fee0822ad2bb732e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,899 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=5baf9d0bdd8c65c757ad796f93bef122, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,43415,1731736285705 2024-11-16T05:51:29,899 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 54bd8d95e5df76bdd2472b3a67cd2b8b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72834971, jitterRate=0.08532564342021942}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:51:29,900 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 54bd8d95e5df76bdd2472b3a67cd2b8b 2024-11-16T05:51:29,900 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 54bd8d95e5df76bdd2472b3a67cd2b8b: Running coprocessor pre-open hook at 1731736289859Writing region info on filesystem at 1731736289859Initializing all the Stores at 1731736289861 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289861Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289862 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289862Cleaning up temporary data from old regions at 1731736289886 (+24 ms)Running coprocessor post-open hooks at 1731736289900 (+14 ms)Region opened successfully at 1731736289900 2024-11-16T05:51:29,900 INFO [StoreOpener-ca14297d921e6ca3fee0822ad2bb732e-1 {}] regionserver.HStore(327): Store=ca14297d921e6ca3fee0822ad2bb732e/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,900 INFO [StoreOpener-ca14297d921e6ca3fee0822ad2bb732e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region ca14297d921e6ca3fee0822ad2bb732e 2024-11-16T05:51:29,902 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731736288173.54bd8d95e5df76bdd2472b3a67cd2b8b., pid=47, masterSystemTime=1731736289710 2024-11-16T05:51:29,903 INFO [StoreOpener-bd79de8195c0452a334d7da789e8f23b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region bd79de8195c0452a334d7da789e8f23b 2024-11-16T05:51:29,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5baf9d0bdd8c65c757ad796f93bef122, server=3456ee6a3164,43415,1731736285705 because future has completed 2024-11-16T05:51:29,904 INFO [StoreOpener-ca14297d921e6ca3fee0822ad2bb732e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca14297d921e6ca3fee0822ad2bb732e columnFamilyName colfamily21 2024-11-16T05:51:29,904 DEBUG [StoreOpener-ca14297d921e6ca3fee0822ad2bb732e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,905 INFO [StoreOpener-ca14297d921e6ca3fee0822ad2bb732e-1 {}] regionserver.HStore(327): Store=ca14297d921e6ca3fee0822ad2bb732e/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,906 INFO [StoreOpener-ca14297d921e6ca3fee0822ad2bb732e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region ca14297d921e6ca3fee0822ad2bb732e 2024-11-16T05:51:29,906 INFO [StoreOpener-bd79de8195c0452a334d7da789e8f23b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd79de8195c0452a334d7da789e8f23b columnFamilyName colfamily11 2024-11-16T05:51:29,906 DEBUG [StoreOpener-bd79de8195c0452a334d7da789e8f23b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,906 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731736288173.54bd8d95e5df76bdd2472b3a67cd2b8b. 2024-11-16T05:51:29,907 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731736288173.54bd8d95e5df76bdd2472b3a67cd2b8b. 2024-11-16T05:51:29,907 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731736288173.afd4d1a884eb847b35b730a1b7a90e0d. 2024-11-16T05:51:29,907 INFO [StoreOpener-bd79de8195c0452a334d7da789e8f23b-1 {}] regionserver.HStore(327): Store=bd79de8195c0452a334d7da789e8f23b/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,907 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => afd4d1a884eb847b35b730a1b7a90e0d, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731736288173.afd4d1a884eb847b35b730a1b7a90e0d.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-16T05:51:29,907 INFO [StoreOpener-bd79de8195c0452a334d7da789e8f23b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region bd79de8195c0452a334d7da789e8f23b 2024-11-16T05:51:29,908 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true afd4d1a884eb847b35b730a1b7a90e0d 2024-11-16T05:51:29,908 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731736288173.afd4d1a884eb847b35b730a1b7a90e0d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:29,908 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for afd4d1a884eb847b35b730a1b7a90e0d 2024-11-16T05:51:29,908 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for afd4d1a884eb847b35b730a1b7a90e0d 2024-11-16T05:51:29,909 INFO [StoreOpener-ca14297d921e6ca3fee0822ad2bb732e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca14297d921e6ca3fee0822ad2bb732e columnFamilyName colfamily31 2024-11-16T05:51:29,909 DEBUG [StoreOpener-ca14297d921e6ca3fee0822ad2bb732e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,910 INFO [StoreOpener-bd79de8195c0452a334d7da789e8f23b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd79de8195c0452a334d7da789e8f23b columnFamilyName colfamily21 2024-11-16T05:51:29,910 DEBUG [StoreOpener-bd79de8195c0452a334d7da789e8f23b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,910 INFO [StoreOpener-ca14297d921e6ca3fee0822ad2bb732e-1 {}] regionserver.HStore(327): Store=ca14297d921e6ca3fee0822ad2bb732e/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,911 INFO [StoreOpener-afd4d1a884eb847b35b730a1b7a90e0d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region afd4d1a884eb847b35b730a1b7a90e0d 2024-11-16T05:51:29,911 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for ca14297d921e6ca3fee0822ad2bb732e 2024-11-16T05:51:29,911 INFO [StoreOpener-bd79de8195c0452a334d7da789e8f23b-1 {}] regionserver.HStore(327): Store=bd79de8195c0452a334d7da789e8f23b/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,911 INFO [StoreOpener-bd79de8195c0452a334d7da789e8f23b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region bd79de8195c0452a334d7da789e8f23b 2024-11-16T05:51:29,912 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ca14297d921e6ca3fee0822ad2bb732e 2024-11-16T05:51:29,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=11 2024-11-16T05:51:29,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 883c672d1e7de6e08e15c0c42278e66d, server=3456ee6a3164,44781,1731736285630 in 359 msec 2024-11-16T05:51:29,913 INFO [StoreOpener-afd4d1a884eb847b35b730a1b7a90e0d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region afd4d1a884eb847b35b730a1b7a90e0d columnFamilyName colfamily11 2024-11-16T05:51:29,913 DEBUG [StoreOpener-afd4d1a884eb847b35b730a1b7a90e0d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,914 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ca14297d921e6ca3fee0822ad2bb732e 2024-11-16T05:51:29,914 INFO [StoreOpener-afd4d1a884eb847b35b730a1b7a90e0d-1 {}] regionserver.HStore(327): Store=afd4d1a884eb847b35b730a1b7a90e0d/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,914 INFO [StoreOpener-afd4d1a884eb847b35b730a1b7a90e0d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region afd4d1a884eb847b35b730a1b7a90e0d 2024-11-16T05:51:29,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=883c672d1e7de6e08e15c0c42278e66d, ASSIGN in 597 msec 2024-11-16T05:51:29,916 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for ca14297d921e6ca3fee0822ad2bb732e 2024-11-16T05:51:29,916 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for ca14297d921e6ca3fee0822ad2bb732e 2024-11-16T05:51:29,917 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=54bd8d95e5df76bdd2472b3a67cd2b8b, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,43415,1731736285705 2024-11-16T05:51:29,918 INFO [StoreOpener-afd4d1a884eb847b35b730a1b7a90e0d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region afd4d1a884eb847b35b730a1b7a90e0d columnFamilyName colfamily21 2024-11-16T05:51:29,918 DEBUG [StoreOpener-afd4d1a884eb847b35b730a1b7a90e0d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,918 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:51:29,919 INFO [StoreOpener-afd4d1a884eb847b35b730a1b7a90e0d-1 {}] regionserver.HStore(327): Store=afd4d1a884eb847b35b730a1b7a90e0d/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,919 INFO [StoreOpener-bd79de8195c0452a334d7da789e8f23b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd79de8195c0452a334d7da789e8f23b columnFamilyName colfamily31 2024-11-16T05:51:29,919 DEBUG [StoreOpener-bd79de8195c0452a334d7da789e8f23b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,919 INFO [StoreOpener-afd4d1a884eb847b35b730a1b7a90e0d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region afd4d1a884eb847b35b730a1b7a90e0d 2024-11-16T05:51:29,920 INFO [StoreOpener-bd79de8195c0452a334d7da789e8f23b-1 {}] regionserver.HStore(327): Store=bd79de8195c0452a334d7da789e8f23b/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,920 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for bd79de8195c0452a334d7da789e8f23b 2024-11-16T05:51:29,921 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for ca14297d921e6ca3fee0822ad2bb732e 2024-11-16T05:51:29,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 54bd8d95e5df76bdd2472b3a67cd2b8b, server=3456ee6a3164,43415,1731736285705 because future has completed 2024-11-16T05:51:29,921 INFO [StoreOpener-afd4d1a884eb847b35b730a1b7a90e0d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region afd4d1a884eb847b35b730a1b7a90e0d columnFamilyName colfamily31 2024-11-16T05:51:29,922 DEBUG [StoreOpener-afd4d1a884eb847b35b730a1b7a90e0d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,922 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bd79de8195c0452a334d7da789e8f23b 2024-11-16T05:51:29,922 INFO [StoreOpener-afd4d1a884eb847b35b730a1b7a90e0d-1 {}] regionserver.HStore(327): Store=afd4d1a884eb847b35b730a1b7a90e0d/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,923 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bd79de8195c0452a334d7da789e8f23b 2024-11-16T05:51:29,923 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for afd4d1a884eb847b35b730a1b7a90e0d 2024-11-16T05:51:29,924 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/afd4d1a884eb847b35b730a1b7a90e0d 2024-11-16T05:51:29,925 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for bd79de8195c0452a334d7da789e8f23b 2024-11-16T05:51:29,925 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ca14297d921e6ca3fee0822ad2bb732e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:51:29,925 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for bd79de8195c0452a334d7da789e8f23b 2024-11-16T05:51:29,925 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/afd4d1a884eb847b35b730a1b7a90e0d 2024-11-16T05:51:29,925 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:51:29,926 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened ca14297d921e6ca3fee0822ad2bb732e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62979463, jitterRate=-0.061532869935035706}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:51:29,926 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ca14297d921e6ca3fee0822ad2bb732e 2024-11-16T05:51:29,926 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for ca14297d921e6ca3fee0822ad2bb732e: Running coprocessor pre-open hook at 1731736289873Writing region info on filesystem at 1731736289873Initializing all the Stores at 1731736289875 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289875Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289895 (+20 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289896 (+1 ms)Cleaning up temporary data from old regions at 1731736289916 (+20 ms)Running coprocessor post-open hooks at 1731736289926 (+10 ms)Region opened successfully at 1731736289926 2024-11-16T05:51:29,927 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for afd4d1a884eb847b35b730a1b7a90e0d 2024-11-16T05:51:29,927 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for afd4d1a884eb847b35b730a1b7a90e0d 2024-11-16T05:51:29,928 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731736288173.ca14297d921e6ca3fee0822ad2bb732e., pid=46, masterSystemTime=1731736289752 2024-11-16T05:51:29,928 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:51:29,929 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for bd79de8195c0452a334d7da789e8f23b 2024-11-16T05:51:29,930 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for afd4d1a884eb847b35b730a1b7a90e0d 2024-11-16T05:51:29,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=14 2024-11-16T05:51:29,931 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 5baf9d0bdd8c65c757ad796f93bef122, server=3456ee6a3164,43415,1731736285705 in 308 msec 2024-11-16T05:51:29,931 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731736288173.ca14297d921e6ca3fee0822ad2bb732e. 2024-11-16T05:51:29,931 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731736288173.ca14297d921e6ca3fee0822ad2bb732e. 2024-11-16T05:51:29,931 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731736288173.63baab6432268e686bc2f01a98a6c75f. 2024-11-16T05:51:29,931 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 63baab6432268e686bc2f01a98a6c75f, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731736288173.63baab6432268e686bc2f01a98a6c75f.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-16T05:51:29,932 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 63baab6432268e686bc2f01a98a6c75f 2024-11-16T05:51:29,932 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731736288173.63baab6432268e686bc2f01a98a6c75f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:29,932 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 63baab6432268e686bc2f01a98a6c75f 2024-11-16T05:51:29,932 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 63baab6432268e686bc2f01a98a6c75f 2024-11-16T05:51:29,935 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/afd4d1a884eb847b35b730a1b7a90e0d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:51:29,935 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened afd4d1a884eb847b35b730a1b7a90e0d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63330931, jitterRate=-0.05629558861255646}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:51:29,936 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for afd4d1a884eb847b35b730a1b7a90e0d 2024-11-16T05:51:29,936 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for afd4d1a884eb847b35b730a1b7a90e0d: Running coprocessor pre-open hook at 1731736289908Writing region info on filesystem at 1731736289908Initializing all the Stores at 1731736289910 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289910Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289910Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289910Cleaning up temporary data from old regions at 1731736289928 (+18 ms)Running coprocessor post-open hooks at 1731736289936 (+8 ms)Region opened successfully at 1731736289936 2024-11-16T05:51:29,936 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bd79de8195c0452a334d7da789e8f23b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:51:29,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=5baf9d0bdd8c65c757ad796f93bef122, ASSIGN in 615 msec 2024-11-16T05:51:29,937 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened bd79de8195c0452a334d7da789e8f23b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61996037, jitterRate=-0.07618705928325653}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:51:29,937 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bd79de8195c0452a334d7da789e8f23b 2024-11-16T05:51:29,937 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for bd79de8195c0452a334d7da789e8f23b: Running coprocessor pre-open hook at 1731736289897Writing region info on filesystem at 1731736289897Initializing all the Stores at 1731736289902 (+5 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289902Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289903 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289903Cleaning up temporary data from old regions at 1731736289925 (+22 ms)Running coprocessor post-open hooks at 1731736289937 (+12 ms)Region opened successfully at 1731736289937 2024-11-16T05:51:29,938 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731736288173.afd4d1a884eb847b35b730a1b7a90e0d., pid=41, masterSystemTime=1731736289710 2024-11-16T05:51:29,938 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b., pid=32, masterSystemTime=1731736289653 2024-11-16T05:51:29,939 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=ca14297d921e6ca3fee0822ad2bb732e, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,33007,1731736285465 2024-11-16T05:51:29,941 INFO [StoreOpener-63baab6432268e686bc2f01a98a6c75f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 63baab6432268e686bc2f01a98a6c75f 2024-11-16T05:51:29,942 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b. 2024-11-16T05:51:29,942 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b. 2024-11-16T05:51:29,942 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00. 2024-11-16T05:51:29,943 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => ec037e054803eda72bad2d81c6143b00, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-16T05:51:29,943 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true ec037e054803eda72bad2d81c6143b00 2024-11-16T05:51:29,943 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:29,944 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for ec037e054803eda72bad2d81c6143b00 2024-11-16T05:51:29,944 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=bd79de8195c0452a334d7da789e8f23b, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,44781,1731736285630 2024-11-16T05:51:29,944 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for ec037e054803eda72bad2d81c6143b00 2024-11-16T05:51:29,944 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731736288173.afd4d1a884eb847b35b730a1b7a90e0d. 2024-11-16T05:51:29,944 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731736288173.afd4d1a884eb847b35b730a1b7a90e0d. 2024-11-16T05:51:29,944 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure ca14297d921e6ca3fee0822ad2bb732e, server=3456ee6a3164,33007,1731736285465 because future has completed 2024-11-16T05:51:29,944 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731736288173.53faee5a89035a5faf9a235d93b486c9. 2024-11-16T05:51:29,944 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 53faee5a89035a5faf9a235d93b486c9, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731736288173.53faee5a89035a5faf9a235d93b486c9.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-16T05:51:29,945 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 53faee5a89035a5faf9a235d93b486c9 2024-11-16T05:51:29,945 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731736288173.53faee5a89035a5faf9a235d93b486c9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:29,945 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 53faee5a89035a5faf9a235d93b486c9 2024-11-16T05:51:29,945 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 53faee5a89035a5faf9a235d93b486c9 2024-11-16T05:51:29,946 INFO [StoreOpener-63baab6432268e686bc2f01a98a6c75f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63baab6432268e686bc2f01a98a6c75f columnFamilyName colfamily11 2024-11-16T05:51:29,946 DEBUG [StoreOpener-63baab6432268e686bc2f01a98a6c75f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,947 INFO [StoreOpener-63baab6432268e686bc2f01a98a6c75f-1 {}] regionserver.HStore(327): Store=63baab6432268e686bc2f01a98a6c75f/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,948 INFO [StoreOpener-63baab6432268e686bc2f01a98a6c75f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 63baab6432268e686bc2f01a98a6c75f 2024-11-16T05:51:29,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd79de8195c0452a334d7da789e8f23b, server=3456ee6a3164,44781,1731736285630 because future has completed 2024-11-16T05:51:29,948 INFO [StoreOpener-53faee5a89035a5faf9a235d93b486c9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 53faee5a89035a5faf9a235d93b486c9 2024-11-16T05:51:29,949 INFO [StoreOpener-ec037e054803eda72bad2d81c6143b00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region ec037e054803eda72bad2d81c6143b00 2024-11-16T05:51:29,952 INFO [StoreOpener-63baab6432268e686bc2f01a98a6c75f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63baab6432268e686bc2f01a98a6c75f columnFamilyName colfamily21 2024-11-16T05:51:29,952 DEBUG [StoreOpener-63baab6432268e686bc2f01a98a6c75f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,953 INFO [StoreOpener-ec037e054803eda72bad2d81c6143b00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec037e054803eda72bad2d81c6143b00 columnFamilyName colfamily11 2024-11-16T05:51:29,953 INFO [StoreOpener-53faee5a89035a5faf9a235d93b486c9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 53faee5a89035a5faf9a235d93b486c9 columnFamilyName colfamily11 2024-11-16T05:51:29,953 DEBUG [StoreOpener-53faee5a89035a5faf9a235d93b486c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,953 DEBUG [StoreOpener-ec037e054803eda72bad2d81c6143b00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,953 INFO [StoreOpener-63baab6432268e686bc2f01a98a6c75f-1 {}] regionserver.HStore(327): Store=63baab6432268e686bc2f01a98a6c75f/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,954 INFO [StoreOpener-53faee5a89035a5faf9a235d93b486c9-1 {}] regionserver.HStore(327): Store=53faee5a89035a5faf9a235d93b486c9/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,954 INFO [StoreOpener-63baab6432268e686bc2f01a98a6c75f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 63baab6432268e686bc2f01a98a6c75f 2024-11-16T05:51:29,954 INFO [StoreOpener-ec037e054803eda72bad2d81c6143b00-1 {}] regionserver.HStore(327): Store=ec037e054803eda72bad2d81c6143b00/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,954 INFO [StoreOpener-ec037e054803eda72bad2d81c6143b00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region ec037e054803eda72bad2d81c6143b00 2024-11-16T05:51:29,954 INFO [StoreOpener-53faee5a89035a5faf9a235d93b486c9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 53faee5a89035a5faf9a235d93b486c9 2024-11-16T05:51:29,957 INFO [StoreOpener-63baab6432268e686bc2f01a98a6c75f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 63baab6432268e686bc2f01a98a6c75f columnFamilyName colfamily31 2024-11-16T05:51:29,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=30 2024-11-16T05:51:29,957 DEBUG [StoreOpener-63baab6432268e686bc2f01a98a6c75f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure ca14297d921e6ca3fee0822ad2bb732e, server=3456ee6a3164,33007,1731736285465 in 348 msec 2024-11-16T05:51:29,958 INFO [StoreOpener-ec037e054803eda72bad2d81c6143b00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec037e054803eda72bad2d81c6143b00 columnFamilyName colfamily21 2024-11-16T05:51:29,958 DEBUG [StoreOpener-ec037e054803eda72bad2d81c6143b00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,958 INFO [StoreOpener-53faee5a89035a5faf9a235d93b486c9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 53faee5a89035a5faf9a235d93b486c9 columnFamilyName colfamily21 2024-11-16T05:51:29,958 DEBUG [StoreOpener-53faee5a89035a5faf9a235d93b486c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,958 INFO [StoreOpener-63baab6432268e686bc2f01a98a6c75f-1 {}] regionserver.HStore(327): Store=63baab6432268e686bc2f01a98a6c75f/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,959 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 63baab6432268e686bc2f01a98a6c75f 2024-11-16T05:51:29,959 INFO [StoreOpener-ec037e054803eda72bad2d81c6143b00-1 {}] regionserver.HStore(327): Store=ec037e054803eda72bad2d81c6143b00/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,959 INFO [StoreOpener-ec037e054803eda72bad2d81c6143b00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region ec037e054803eda72bad2d81c6143b00 2024-11-16T05:51:29,960 INFO [StoreOpener-53faee5a89035a5faf9a235d93b486c9-1 {}] regionserver.HStore(327): Store=53faee5a89035a5faf9a235d93b486c9/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,960 INFO [StoreOpener-53faee5a89035a5faf9a235d93b486c9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 53faee5a89035a5faf9a235d93b486c9 2024-11-16T05:51:29,960 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/63baab6432268e686bc2f01a98a6c75f 2024-11-16T05:51:29,961 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/63baab6432268e686bc2f01a98a6c75f 2024-11-16T05:51:29,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ca14297d921e6ca3fee0822ad2bb732e, ASSIGN in 642 msec 2024-11-16T05:51:29,963 INFO [StoreOpener-ec037e054803eda72bad2d81c6143b00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec037e054803eda72bad2d81c6143b00 columnFamilyName colfamily31 2024-11-16T05:51:29,963 INFO [StoreOpener-53faee5a89035a5faf9a235d93b486c9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 53faee5a89035a5faf9a235d93b486c9 columnFamilyName colfamily31 2024-11-16T05:51:29,963 DEBUG [StoreOpener-ec037e054803eda72bad2d81c6143b00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,963 DEBUG [StoreOpener-53faee5a89035a5faf9a235d93b486c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:29,964 INFO [StoreOpener-ec037e054803eda72bad2d81c6143b00-1 {}] regionserver.HStore(327): Store=ec037e054803eda72bad2d81c6143b00/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,964 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 63baab6432268e686bc2f01a98a6c75f 2024-11-16T05:51:29,965 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 63baab6432268e686bc2f01a98a6c75f 2024-11-16T05:51:29,965 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for ec037e054803eda72bad2d81c6143b00 2024-11-16T05:51:29,965 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44685 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3456ee6a3164,33007,1731736285465, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3e30199c6045096f8f565c93af9b6088. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-16T05:51:29,965 INFO [StoreOpener-53faee5a89035a5faf9a235d93b486c9-1 {}] regionserver.HStore(327): Store=53faee5a89035a5faf9a235d93b486c9/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:29,966 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 53faee5a89035a5faf9a235d93b486c9 2024-11-16T05:51:29,966 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:51:29,967 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ec037e054803eda72bad2d81c6143b00 2024-11-16T05:51:29,969 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/53faee5a89035a5faf9a235d93b486c9 2024-11-16T05:51:29,969 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ec037e054803eda72bad2d81c6143b00 2024-11-16T05:51:29,969 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44685 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3456ee6a3164,43415,1731736285705, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=54bd8d95e5df76bdd2472b3a67cd2b8b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-16T05:51:29,970 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/53faee5a89035a5faf9a235d93b486c9 2024-11-16T05:51:29,971 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 63baab6432268e686bc2f01a98a6c75f 2024-11-16T05:51:29,971 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for ec037e054803eda72bad2d81c6143b00 2024-11-16T05:51:29,971 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for ec037e054803eda72bad2d81c6143b00 2024-11-16T05:51:29,972 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 53faee5a89035a5faf9a235d93b486c9 2024-11-16T05:51:29,972 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:51:29,972 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 53faee5a89035a5faf9a235d93b486c9 2024-11-16T05:51:29,973 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:51:29,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=17 2024-11-16T05:51:29,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure bd79de8195c0452a334d7da789e8f23b, server=3456ee6a3164,44781,1731736285630 in 456 msec 2024-11-16T05:51:29,975 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for ec037e054803eda72bad2d81c6143b00 2024-11-16T05:51:29,976 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 53faee5a89035a5faf9a235d93b486c9 2024-11-16T05:51:29,976 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/63baab6432268e686bc2f01a98a6c75f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:51:29,977 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 63baab6432268e686bc2f01a98a6c75f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70735655, jitterRate=0.054043397307395935}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:51:29,977 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 63baab6432268e686bc2f01a98a6c75f 2024-11-16T05:51:29,978 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 63baab6432268e686bc2f01a98a6c75f: Running coprocessor pre-open hook at 1731736289932Writing region info on filesystem at 1731736289932Initializing all the Stores at 1731736289934 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289934Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289941 (+7 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289941Cleaning up temporary data from old regions at 1731736289965 (+24 ms)Running coprocessor post-open hooks at 1731736289977 (+12 ms)Region opened successfully at 1731736289977 2024-11-16T05:51:29,979 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731736288173.63baab6432268e686bc2f01a98a6c75f., pid=51, masterSystemTime=1731736289752 2024-11-16T05:51:29,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bd79de8195c0452a334d7da789e8f23b, ASSIGN in 658 msec 2024-11-16T05:51:29,979 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ec037e054803eda72bad2d81c6143b00/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:51:29,984 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened ec037e054803eda72bad2d81c6143b00; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73202846, jitterRate=0.09080740809440613}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:51:29,984 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ec037e054803eda72bad2d81c6143b00 2024-11-16T05:51:29,984 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for ec037e054803eda72bad2d81c6143b00: Running coprocessor pre-open hook at 1731736289944Writing region info on filesystem at 1731736289944Initializing all the Stores at 1731736289948 (+4 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289949 (+1 ms)Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289949Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289949Cleaning up temporary data from old regions at 1731736289971 (+22 ms)Running coprocessor post-open hooks at 1731736289984 (+13 ms)Region opened successfully at 1731736289984 2024-11-16T05:51:29,985 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731736288173.63baab6432268e686bc2f01a98a6c75f. 2024-11-16T05:51:29,985 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=afd4d1a884eb847b35b730a1b7a90e0d, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,43415,1731736285705 2024-11-16T05:51:29,985 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731736288173.63baab6432268e686bc2f01a98a6c75f. 2024-11-16T05:51:29,985 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731736288173.eb73dbcec883aba378443bd58f1eb724. 2024-11-16T05:51:29,985 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => eb73dbcec883aba378443bd58f1eb724, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731736288173.eb73dbcec883aba378443bd58f1eb724.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-16T05:51:29,986 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true eb73dbcec883aba378443bd58f1eb724 2024-11-16T05:51:29,986 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731736288173.eb73dbcec883aba378443bd58f1eb724.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:29,986 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for eb73dbcec883aba378443bd58f1eb724 2024-11-16T05:51:29,986 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for eb73dbcec883aba378443bd58f1eb724 2024-11-16T05:51:29,989 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00., pid=34, masterSystemTime=1731736289653 2024-11-16T05:51:29,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure afd4d1a884eb847b35b730a1b7a90e0d, server=3456ee6a3164,43415,1731736285705 because future has completed 2024-11-16T05:51:29,993 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=63baab6432268e686bc2f01a98a6c75f, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,33007,1731736285465 2024-11-16T05:51:29,993 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00. 2024-11-16T05:51:29,993 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/53faee5a89035a5faf9a235d93b486c9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:51:29,993 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00. 2024-11-16T05:51:29,994 INFO [StoreOpener-eb73dbcec883aba378443bd58f1eb724-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region eb73dbcec883aba378443bd58f1eb724 2024-11-16T05:51:29,995 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 53faee5a89035a5faf9a235d93b486c9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70471626, jitterRate=0.05010905861854553}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:51:29,996 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 53faee5a89035a5faf9a235d93b486c9 2024-11-16T05:51:29,996 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 53faee5a89035a5faf9a235d93b486c9: Running coprocessor pre-open hook at 1731736289946Writing region info on filesystem at 1731736289946Initializing all the Stores at 1731736289948 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289948Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289948Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289948Cleaning up temporary data from old regions at 1731736289972 (+24 ms)Running coprocessor post-open hooks at 1731736289996 (+24 ms)Region opened successfully at 1731736289996 2024-11-16T05:51:29,997 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 63baab6432268e686bc2f01a98a6c75f, server=3456ee6a3164,33007,1731736285465 because future has completed 2024-11-16T05:51:29,998 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731736288173.53faee5a89035a5faf9a235d93b486c9., pid=44, masterSystemTime=1731736289710 2024-11-16T05:51:29,998 INFO [StoreOpener-eb73dbcec883aba378443bd58f1eb724-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb73dbcec883aba378443bd58f1eb724 columnFamilyName colfamily11 2024-11-16T05:51:29,998 DEBUG [StoreOpener-eb73dbcec883aba378443bd58f1eb724-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:30,001 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731736288173.53faee5a89035a5faf9a235d93b486c9. 2024-11-16T05:51:30,001 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731736288173.53faee5a89035a5faf9a235d93b486c9. 2024-11-16T05:51:30,001 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731736288173.2beaf8caee4e552158f6bb091a0437bb. 2024-11-16T05:51:30,001 INFO [StoreOpener-eb73dbcec883aba378443bd58f1eb724-1 {}] regionserver.HStore(327): Store=eb73dbcec883aba378443bd58f1eb724/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:30,002 INFO [StoreOpener-eb73dbcec883aba378443bd58f1eb724-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region eb73dbcec883aba378443bd58f1eb724 2024-11-16T05:51:30,002 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 2beaf8caee4e552158f6bb091a0437bb, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731736288173.2beaf8caee4e552158f6bb091a0437bb.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-16T05:51:30,002 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2beaf8caee4e552158f6bb091a0437bb 2024-11-16T05:51:30,002 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731736288173.2beaf8caee4e552158f6bb091a0437bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:30,002 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 2beaf8caee4e552158f6bb091a0437bb 2024-11-16T05:51:30,003 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 2beaf8caee4e552158f6bb091a0437bb 2024-11-16T05:51:30,004 INFO [StoreOpener-eb73dbcec883aba378443bd58f1eb724-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb73dbcec883aba378443bd58f1eb724 columnFamilyName colfamily21 2024-11-16T05:51:30,005 DEBUG [StoreOpener-eb73dbcec883aba378443bd58f1eb724-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:30,005 INFO [StoreOpener-2beaf8caee4e552158f6bb091a0437bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 2beaf8caee4e552158f6bb091a0437bb 2024-11-16T05:51:30,006 INFO [StoreOpener-eb73dbcec883aba378443bd58f1eb724-1 {}] regionserver.HStore(327): Store=eb73dbcec883aba378443bd58f1eb724/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:30,006 INFO [StoreOpener-eb73dbcec883aba378443bd58f1eb724-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region eb73dbcec883aba378443bd58f1eb724 2024-11-16T05:51:30,009 INFO [StoreOpener-eb73dbcec883aba378443bd58f1eb724-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb73dbcec883aba378443bd58f1eb724 columnFamilyName colfamily31 2024-11-16T05:51:30,009 DEBUG [StoreOpener-eb73dbcec883aba378443bd58f1eb724-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:30,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=15 2024-11-16T05:51:30,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 63baab6432268e686bc2f01a98a6c75f, server=3456ee6a3164,33007,1731736285465 in 373 msec 2024-11-16T05:51:30,010 INFO [StoreOpener-2beaf8caee4e552158f6bb091a0437bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2beaf8caee4e552158f6bb091a0437bb columnFamilyName colfamily11 2024-11-16T05:51:30,010 DEBUG [StoreOpener-2beaf8caee4e552158f6bb091a0437bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:30,010 INFO [StoreOpener-eb73dbcec883aba378443bd58f1eb724-1 {}] regionserver.HStore(327): Store=eb73dbcec883aba378443bd58f1eb724/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:30,010 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for eb73dbcec883aba378443bd58f1eb724 2024-11-16T05:51:30,012 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/eb73dbcec883aba378443bd58f1eb724 2024-11-16T05:51:30,012 INFO [StoreOpener-2beaf8caee4e552158f6bb091a0437bb-1 {}] regionserver.HStore(327): Store=2beaf8caee4e552158f6bb091a0437bb/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:30,013 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/eb73dbcec883aba378443bd58f1eb724 2024-11-16T05:51:30,012 INFO [StoreOpener-2beaf8caee4e552158f6bb091a0437bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 2beaf8caee4e552158f6bb091a0437bb 2024-11-16T05:51:30,013 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=63baab6432268e686bc2f01a98a6c75f, ASSIGN in 694 msec 2024-11-16T05:51:30,014 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for eb73dbcec883aba378443bd58f1eb724 2024-11-16T05:51:30,014 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for eb73dbcec883aba378443bd58f1eb724 2024-11-16T05:51:30,016 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:51:30,016 INFO [StoreOpener-2beaf8caee4e552158f6bb091a0437bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2beaf8caee4e552158f6bb091a0437bb columnFamilyName colfamily21 2024-11-16T05:51:30,016 DEBUG [StoreOpener-2beaf8caee4e552158f6bb091a0437bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:30,017 INFO [StoreOpener-2beaf8caee4e552158f6bb091a0437bb-1 {}] regionserver.HStore(327): Store=2beaf8caee4e552158f6bb091a0437bb/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:30,017 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=53faee5a89035a5faf9a235d93b486c9, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,43415,1731736285705 2024-11-16T05:51:30,017 INFO [StoreOpener-2beaf8caee4e552158f6bb091a0437bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 2beaf8caee4e552158f6bb091a0437bb 2024-11-16T05:51:30,018 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for eb73dbcec883aba378443bd58f1eb724 2024-11-16T05:51:30,019 INFO [StoreOpener-2beaf8caee4e552158f6bb091a0437bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2beaf8caee4e552158f6bb091a0437bb columnFamilyName colfamily31 2024-11-16T05:51:30,019 DEBUG [StoreOpener-2beaf8caee4e552158f6bb091a0437bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:30,021 INFO [StoreOpener-2beaf8caee4e552158f6bb091a0437bb-1 {}] regionserver.HStore(327): Store=2beaf8caee4e552158f6bb091a0437bb/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:30,021 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 2beaf8caee4e552158f6bb091a0437bb 2024-11-16T05:51:30,021 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 53faee5a89035a5faf9a235d93b486c9, server=3456ee6a3164,43415,1731736285705 because future has completed 2024-11-16T05:51:30,023 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2beaf8caee4e552158f6bb091a0437bb 2024-11-16T05:51:30,023 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2beaf8caee4e552158f6bb091a0437bb 2024-11-16T05:51:30,024 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/eb73dbcec883aba378443bd58f1eb724/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:51:30,024 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened eb73dbcec883aba378443bd58f1eb724; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62845565, jitterRate=-0.06352810561656952}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:51:30,024 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eb73dbcec883aba378443bd58f1eb724 2024-11-16T05:51:30,025 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for eb73dbcec883aba378443bd58f1eb724: Running coprocessor pre-open hook at 1731736289986Writing region info on filesystem at 1731736289986Initializing all the Stores at 1731736289988 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289988Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289994 (+6 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736289994Cleaning up temporary data from old regions at 1731736290014 (+20 ms)Running coprocessor post-open hooks at 1731736290025 (+11 ms)Region opened successfully at 1731736290025 2024-11-16T05:51:30,025 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 2beaf8caee4e552158f6bb091a0437bb 2024-11-16T05:51:30,025 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 2beaf8caee4e552158f6bb091a0437bb 2024-11-16T05:51:30,026 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:51:30,026 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731736288173.eb73dbcec883aba378443bd58f1eb724., pid=45, masterSystemTime=1731736289752 2024-11-16T05:51:30,030 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 2beaf8caee4e552158f6bb091a0437bb 2024-11-16T05:51:30,030 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731736288173.eb73dbcec883aba378443bd58f1eb724. 2024-11-16T05:51:30,030 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731736288173.eb73dbcec883aba378443bd58f1eb724. 2024-11-16T05:51:30,031 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731736288173.12c6ab39e16284ff5b8ffeb36e672416. 2024-11-16T05:51:30,031 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 12c6ab39e16284ff5b8ffeb36e672416, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731736288173.12c6ab39e16284ff5b8ffeb36e672416.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-16T05:51:30,031 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 12c6ab39e16284ff5b8ffeb36e672416 2024-11-16T05:51:30,032 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731736288173.12c6ab39e16284ff5b8ffeb36e672416.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:30,032 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 12c6ab39e16284ff5b8ffeb36e672416 2024-11-16T05:51:30,032 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 12c6ab39e16284ff5b8ffeb36e672416 2024-11-16T05:51:30,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=9 2024-11-16T05:51:30,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 53faee5a89035a5faf9a235d93b486c9, server=3456ee6a3164,43415,1731736285705 in 432 msec 2024-11-16T05:51:30,035 INFO [StoreOpener-12c6ab39e16284ff5b8ffeb36e672416-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 12c6ab39e16284ff5b8ffeb36e672416 2024-11-16T05:51:30,035 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2beaf8caee4e552158f6bb091a0437bb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:51:30,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=53faee5a89035a5faf9a235d93b486c9, ASSIGN in 717 msec 2024-11-16T05:51:30,037 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 2beaf8caee4e552158f6bb091a0437bb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66446441, jitterRate=-0.009870871901512146}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:51:30,037 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2beaf8caee4e552158f6bb091a0437bb 2024-11-16T05:51:30,037 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 2beaf8caee4e552158f6bb091a0437bb: Running coprocessor pre-open hook at 1731736290003Writing region info on filesystem at 1731736290003Initializing all the Stores at 1731736290004 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736290005 (+1 ms)Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736290005Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736290005Cleaning up temporary data from old regions at 1731736290025 (+20 ms)Running coprocessor post-open hooks at 1731736290037 (+12 ms)Region opened successfully at 1731736290037 2024-11-16T05:51:30,038 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=eb73dbcec883aba378443bd58f1eb724, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,33007,1731736285465 2024-11-16T05:51:30,039 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731736288173.2beaf8caee4e552158f6bb091a0437bb., pid=40, masterSystemTime=1731736289710 2024-11-16T05:51:30,040 INFO [StoreOpener-12c6ab39e16284ff5b8ffeb36e672416-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12c6ab39e16284ff5b8ffeb36e672416 columnFamilyName colfamily11 2024-11-16T05:51:30,040 DEBUG [StoreOpener-12c6ab39e16284ff5b8ffeb36e672416-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:30,041 INFO [StoreOpener-12c6ab39e16284ff5b8ffeb36e672416-1 {}] regionserver.HStore(327): Store=12c6ab39e16284ff5b8ffeb36e672416/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:30,041 INFO [StoreOpener-12c6ab39e16284ff5b8ffeb36e672416-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 12c6ab39e16284ff5b8ffeb36e672416 2024-11-16T05:51:30,043 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb73dbcec883aba378443bd58f1eb724, server=3456ee6a3164,33007,1731736285465 because future has completed 2024-11-16T05:51:30,044 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=ec037e054803eda72bad2d81c6143b00, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,44781,1731736285630 2024-11-16T05:51:30,044 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731736288173.2beaf8caee4e552158f6bb091a0437bb. 2024-11-16T05:51:30,044 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731736288173.2beaf8caee4e552158f6bb091a0437bb. 2024-11-16T05:51:30,044 INFO [StoreOpener-12c6ab39e16284ff5b8ffeb36e672416-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12c6ab39e16284ff5b8ffeb36e672416 columnFamilyName colfamily21 2024-11-16T05:51:30,044 DEBUG [StoreOpener-12c6ab39e16284ff5b8ffeb36e672416-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:30,044 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731736288173.408fc7812ce9cee6ea99428f155ce5a8. 2024-11-16T05:51:30,045 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => 408fc7812ce9cee6ea99428f155ce5a8, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731736288173.408fc7812ce9cee6ea99428f155ce5a8.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-16T05:51:30,045 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 408fc7812ce9cee6ea99428f155ce5a8 2024-11-16T05:51:30,045 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731736288173.408fc7812ce9cee6ea99428f155ce5a8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:30,045 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for 408fc7812ce9cee6ea99428f155ce5a8 2024-11-16T05:51:30,045 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for 408fc7812ce9cee6ea99428f155ce5a8 2024-11-16T05:51:30,045 INFO [StoreOpener-12c6ab39e16284ff5b8ffeb36e672416-1 {}] regionserver.HStore(327): Store=12c6ab39e16284ff5b8ffeb36e672416/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:30,046 INFO [StoreOpener-12c6ab39e16284ff5b8ffeb36e672416-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 12c6ab39e16284ff5b8ffeb36e672416 2024-11-16T05:51:30,048 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure ec037e054803eda72bad2d81c6143b00, server=3456ee6a3164,44781,1731736285630 because future has completed 2024-11-16T05:51:30,049 INFO [StoreOpener-12c6ab39e16284ff5b8ffeb36e672416-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12c6ab39e16284ff5b8ffeb36e672416 columnFamilyName colfamily31 2024-11-16T05:51:30,049 INFO [StoreOpener-408fc7812ce9cee6ea99428f155ce5a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 408fc7812ce9cee6ea99428f155ce5a8 2024-11-16T05:51:30,049 DEBUG [StoreOpener-12c6ab39e16284ff5b8ffeb36e672416-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:30,049 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=2beaf8caee4e552158f6bb091a0437bb, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,43415,1731736285705 2024-11-16T05:51:30,050 INFO [StoreOpener-12c6ab39e16284ff5b8ffeb36e672416-1 {}] regionserver.HStore(327): Store=12c6ab39e16284ff5b8ffeb36e672416/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:30,050 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 12c6ab39e16284ff5b8ffeb36e672416 2024-11-16T05:51:30,052 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/12c6ab39e16284ff5b8ffeb36e672416 2024-11-16T05:51:30,053 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2beaf8caee4e552158f6bb091a0437bb, server=3456ee6a3164,43415,1731736285705 because future has completed 2024-11-16T05:51:30,053 INFO [StoreOpener-408fc7812ce9cee6ea99428f155ce5a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 408fc7812ce9cee6ea99428f155ce5a8 columnFamilyName colfamily11 2024-11-16T05:51:30,053 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/12c6ab39e16284ff5b8ffeb36e672416 2024-11-16T05:51:30,054 DEBUG [StoreOpener-408fc7812ce9cee6ea99428f155ce5a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:30,054 INFO [StoreOpener-408fc7812ce9cee6ea99428f155ce5a8-1 {}] regionserver.HStore(327): Store=408fc7812ce9cee6ea99428f155ce5a8/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:30,055 INFO [StoreOpener-408fc7812ce9cee6ea99428f155ce5a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 408fc7812ce9cee6ea99428f155ce5a8 2024-11-16T05:51:30,055 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 12c6ab39e16284ff5b8ffeb36e672416 2024-11-16T05:51:30,055 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 12c6ab39e16284ff5b8ffeb36e672416 2024-11-16T05:51:30,056 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:51:30,057 INFO [StoreOpener-408fc7812ce9cee6ea99428f155ce5a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 408fc7812ce9cee6ea99428f155ce5a8 columnFamilyName colfamily21 2024-11-16T05:51:30,057 DEBUG [StoreOpener-408fc7812ce9cee6ea99428f155ce5a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:30,058 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=20 2024-11-16T05:51:30,058 INFO [StoreOpener-408fc7812ce9cee6ea99428f155ce5a8-1 {}] regionserver.HStore(327): Store=408fc7812ce9cee6ea99428f155ce5a8/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:30,058 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure ec037e054803eda72bad2d81c6143b00, server=3456ee6a3164,44781,1731736285630 in 532 msec 2024-11-16T05:51:30,058 INFO [StoreOpener-408fc7812ce9cee6ea99428f155ce5a8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 408fc7812ce9cee6ea99428f155ce5a8 2024-11-16T05:51:30,059 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 12c6ab39e16284ff5b8ffeb36e672416 2024-11-16T05:51:30,061 INFO [StoreOpener-408fc7812ce9cee6ea99428f155ce5a8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 408fc7812ce9cee6ea99428f155ce5a8 columnFamilyName colfamily31 2024-11-16T05:51:30,061 DEBUG [StoreOpener-408fc7812ce9cee6ea99428f155ce5a8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:30,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ec037e054803eda72bad2d81c6143b00, ASSIGN in 742 msec 2024-11-16T05:51:30,062 INFO [StoreOpener-408fc7812ce9cee6ea99428f155ce5a8-1 {}] regionserver.HStore(327): Store=408fc7812ce9cee6ea99428f155ce5a8/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:30,062 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for 408fc7812ce9cee6ea99428f155ce5a8 2024-11-16T05:51:30,064 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/408fc7812ce9cee6ea99428f155ce5a8 2024-11-16T05:51:30,064 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/408fc7812ce9cee6ea99428f155ce5a8 2024-11-16T05:51:30,064 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/12c6ab39e16284ff5b8ffeb36e672416/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:51:30,065 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 12c6ab39e16284ff5b8ffeb36e672416; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60238481, jitterRate=-0.10237668454647064}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:51:30,066 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 12c6ab39e16284ff5b8ffeb36e672416 2024-11-16T05:51:30,066 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 12c6ab39e16284ff5b8ffeb36e672416: Running coprocessor pre-open hook at 1731736290032Writing region info on filesystem at 1731736290032Initializing all the Stores at 1731736290033 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736290033Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736290035 (+2 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736290035Cleaning up temporary data from old regions at 1731736290055 (+20 ms)Running coprocessor post-open hooks at 1731736290066 (+11 ms)Region opened successfully at 1731736290066 2024-11-16T05:51:30,066 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for 408fc7812ce9cee6ea99428f155ce5a8 2024-11-16T05:51:30,066 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for 408fc7812ce9cee6ea99428f155ce5a8 2024-11-16T05:51:30,068 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731736288173.12c6ab39e16284ff5b8ffeb36e672416., pid=55, masterSystemTime=1731736289752 2024-11-16T05:51:30,068 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:51:30,069 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44685 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3456ee6a3164,33007,1731736285465, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3e30199c6045096f8f565c93af9b6088. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-16T05:51:30,070 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44685 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3456ee6a3164,33007,1731736285465, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=eb73dbcec883aba378443bd58f1eb724. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-16T05:51:30,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=24 2024-11-16T05:51:30,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 2beaf8caee4e552158f6bb091a0437bb, server=3456ee6a3164,43415,1731736285705 in 491 msec 2024-11-16T05:51:30,073 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for 408fc7812ce9cee6ea99428f155ce5a8 2024-11-16T05:51:30,076 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44685 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3456ee6a3164,43415,1731736285705, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=54bd8d95e5df76bdd2472b3a67cd2b8b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-16T05:51:30,077 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44685 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3456ee6a3164,43415,1731736285705, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=afd4d1a884eb847b35b730a1b7a90e0d. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-16T05:51:30,077 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/408fc7812ce9cee6ea99428f155ce5a8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:51:30,077 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731736288173.12c6ab39e16284ff5b8ffeb36e672416. 2024-11-16T05:51:30,077 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened 408fc7812ce9cee6ea99428f155ce5a8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70031708, jitterRate=0.04355376958847046}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:51:30,078 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 408fc7812ce9cee6ea99428f155ce5a8 2024-11-16T05:51:30,078 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731736288173.12c6ab39e16284ff5b8ffeb36e672416. 2024-11-16T05:51:30,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2beaf8caee4e552158f6bb091a0437bb, ASSIGN in 755 msec 2024-11-16T05:51:30,078 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for 408fc7812ce9cee6ea99428f155ce5a8: Running coprocessor pre-open hook at 1731736290046Writing region info on filesystem at 1731736290046Initializing all the Stores at 1731736290048 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736290048Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736290048Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736290048Cleaning up temporary data from old regions at 1731736290066 (+18 ms)Running coprocessor post-open hooks at 1731736290078 (+12 ms)Region opened successfully at 1731736290078 2024-11-16T05:51:30,078 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731736288173.9df9b8a2b43462b728e2352478a97347. 2024-11-16T05:51:30,078 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 9df9b8a2b43462b728e2352478a97347, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731736288173.9df9b8a2b43462b728e2352478a97347.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-16T05:51:30,079 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 9df9b8a2b43462b728e2352478a97347 2024-11-16T05:51:30,079 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731736288173.9df9b8a2b43462b728e2352478a97347.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:30,079 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731736288173.408fc7812ce9cee6ea99428f155ce5a8., pid=43, masterSystemTime=1731736289710 2024-11-16T05:51:30,079 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 9df9b8a2b43462b728e2352478a97347 2024-11-16T05:51:30,079 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 9df9b8a2b43462b728e2352478a97347 2024-11-16T05:51:30,079 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=12c6ab39e16284ff5b8ffeb36e672416, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,33007,1731736285465 2024-11-16T05:51:30,083 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731736288173.408fc7812ce9cee6ea99428f155ce5a8. 2024-11-16T05:51:30,083 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731736288173.408fc7812ce9cee6ea99428f155ce5a8. 2024-11-16T05:51:30,083 INFO [StoreOpener-9df9b8a2b43462b728e2352478a97347-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 9df9b8a2b43462b728e2352478a97347 2024-11-16T05:51:30,083 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731736288173.ff61a3918e3c512081ba7b9c1e1a4948. 2024-11-16T05:51:30,083 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => ff61a3918e3c512081ba7b9c1e1a4948, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731736288173.ff61a3918e3c512081ba7b9c1e1a4948.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-16T05:51:30,083 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true ff61a3918e3c512081ba7b9c1e1a4948 2024-11-16T05:51:30,084 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731736288173.ff61a3918e3c512081ba7b9c1e1a4948.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:30,084 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for ff61a3918e3c512081ba7b9c1e1a4948 2024-11-16T05:51:30,084 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for ff61a3918e3c512081ba7b9c1e1a4948 2024-11-16T05:51:30,085 INFO [StoreOpener-9df9b8a2b43462b728e2352478a97347-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9df9b8a2b43462b728e2352478a97347 columnFamilyName colfamily11 2024-11-16T05:51:30,085 DEBUG [StoreOpener-9df9b8a2b43462b728e2352478a97347-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:30,086 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=408fc7812ce9cee6ea99428f155ce5a8, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,43415,1731736285705 2024-11-16T05:51:30,086 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12c6ab39e16284ff5b8ffeb36e672416, server=3456ee6a3164,33007,1731736285465 because future has completed 2024-11-16T05:51:30,089 INFO [StoreOpener-9df9b8a2b43462b728e2352478a97347-1 {}] regionserver.HStore(327): Store=9df9b8a2b43462b728e2352478a97347/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:30,089 INFO [StoreOpener-9df9b8a2b43462b728e2352478a97347-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 9df9b8a2b43462b728e2352478a97347 2024-11-16T05:51:30,091 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 408fc7812ce9cee6ea99428f155ce5a8, server=3456ee6a3164,43415,1731736285705 because future has completed 2024-11-16T05:51:30,092 INFO [StoreOpener-ff61a3918e3c512081ba7b9c1e1a4948-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region ff61a3918e3c512081ba7b9c1e1a4948 2024-11-16T05:51:30,094 INFO [StoreOpener-ff61a3918e3c512081ba7b9c1e1a4948-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff61a3918e3c512081ba7b9c1e1a4948 columnFamilyName colfamily11 2024-11-16T05:51:30,094 DEBUG [StoreOpener-ff61a3918e3c512081ba7b9c1e1a4948-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:30,095 INFO [StoreOpener-9df9b8a2b43462b728e2352478a97347-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9df9b8a2b43462b728e2352478a97347 columnFamilyName colfamily21 2024-11-16T05:51:30,095 DEBUG [StoreOpener-9df9b8a2b43462b728e2352478a97347-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:30,095 INFO [StoreOpener-ff61a3918e3c512081ba7b9c1e1a4948-1 {}] regionserver.HStore(327): Store=ff61a3918e3c512081ba7b9c1e1a4948/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:30,095 INFO [StoreOpener-9df9b8a2b43462b728e2352478a97347-1 {}] regionserver.HStore(327): Store=9df9b8a2b43462b728e2352478a97347/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:30,095 INFO [StoreOpener-ff61a3918e3c512081ba7b9c1e1a4948-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region ff61a3918e3c512081ba7b9c1e1a4948 2024-11-16T05:51:30,096 INFO [StoreOpener-9df9b8a2b43462b728e2352478a97347-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 9df9b8a2b43462b728e2352478a97347 2024-11-16T05:51:30,097 INFO [StoreOpener-ff61a3918e3c512081ba7b9c1e1a4948-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff61a3918e3c512081ba7b9c1e1a4948 columnFamilyName colfamily21 2024-11-16T05:51:30,097 INFO [StoreOpener-9df9b8a2b43462b728e2352478a97347-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9df9b8a2b43462b728e2352478a97347 columnFamilyName colfamily31 2024-11-16T05:51:30,097 DEBUG [StoreOpener-9df9b8a2b43462b728e2352478a97347-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:30,098 DEBUG [StoreOpener-ff61a3918e3c512081ba7b9c1e1a4948-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:30,098 INFO [StoreOpener-ff61a3918e3c512081ba7b9c1e1a4948-1 {}] regionserver.HStore(327): Store=ff61a3918e3c512081ba7b9c1e1a4948/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:30,098 INFO [StoreOpener-9df9b8a2b43462b728e2352478a97347-1 {}] regionserver.HStore(327): Store=9df9b8a2b43462b728e2352478a97347/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:30,098 INFO [StoreOpener-ff61a3918e3c512081ba7b9c1e1a4948-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region ff61a3918e3c512081ba7b9c1e1a4948 2024-11-16T05:51:30,098 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 9df9b8a2b43462b728e2352478a97347 2024-11-16T05:51:30,099 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9df9b8a2b43462b728e2352478a97347 2024-11-16T05:51:30,100 INFO [StoreOpener-ff61a3918e3c512081ba7b9c1e1a4948-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff61a3918e3c512081ba7b9c1e1a4948 columnFamilyName colfamily31 2024-11-16T05:51:30,100 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9df9b8a2b43462b728e2352478a97347 2024-11-16T05:51:30,100 DEBUG [StoreOpener-ff61a3918e3c512081ba7b9c1e1a4948-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:30,101 INFO [StoreOpener-ff61a3918e3c512081ba7b9c1e1a4948-1 {}] regionserver.HStore(327): Store=ff61a3918e3c512081ba7b9c1e1a4948/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:30,102 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for ff61a3918e3c512081ba7b9c1e1a4948 2024-11-16T05:51:30,102 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 9df9b8a2b43462b728e2352478a97347 2024-11-16T05:51:30,102 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 9df9b8a2b43462b728e2352478a97347 2024-11-16T05:51:30,103 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ff61a3918e3c512081ba7b9c1e1a4948 2024-11-16T05:51:30,103 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:51:30,104 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ff61a3918e3c512081ba7b9c1e1a4948 2024-11-16T05:51:30,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=18 2024-11-16T05:51:30,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 12c6ab39e16284ff5b8ffeb36e672416, server=3456ee6a3164,33007,1731736285465 in 450 msec 2024-11-16T05:51:30,105 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for ff61a3918e3c512081ba7b9c1e1a4948 2024-11-16T05:51:30,105 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for ff61a3918e3c512081ba7b9c1e1a4948 2024-11-16T05:51:30,106 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:51:30,107 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 9df9b8a2b43462b728e2352478a97347 2024-11-16T05:51:30,108 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for ff61a3918e3c512081ba7b9c1e1a4948 2024-11-16T05:51:30,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=12c6ab39e16284ff5b8ffeb36e672416, ASSIGN in 788 msec 2024-11-16T05:51:30,111 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9df9b8a2b43462b728e2352478a97347/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:51:30,111 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ff61a3918e3c512081ba7b9c1e1a4948/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:51:30,111 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 9df9b8a2b43462b728e2352478a97347; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69423005, jitterRate=0.03448338806629181}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:51:30,112 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9df9b8a2b43462b728e2352478a97347 2024-11-16T05:51:30,112 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 9df9b8a2b43462b728e2352478a97347: Running coprocessor pre-open hook at 1731736290080Writing region info on filesystem at 1731736290080Initializing all the Stores at 1731736290082 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736290082Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736290082Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736290082Cleaning up temporary data from old regions at 1731736290102 (+20 ms)Running coprocessor post-open hooks at 1731736290112 (+10 ms)Region opened successfully at 1731736290112 2024-11-16T05:51:30,112 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened ff61a3918e3c512081ba7b9c1e1a4948; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73093523, jitterRate=0.08917836844921112}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:51:30,112 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ff61a3918e3c512081ba7b9c1e1a4948 2024-11-16T05:51:30,113 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for ff61a3918e3c512081ba7b9c1e1a4948: Running coprocessor pre-open hook at 1731736290084Writing region info on filesystem at 1731736290084Initializing all the Stores at 1731736290090 (+6 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736290091 (+1 ms)Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736290091Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736290092 (+1 ms)Cleaning up temporary data from old regions at 1731736290105 (+13 ms)Running coprocessor post-open hooks at 1731736290113 (+8 ms)Region opened successfully at 1731736290113 2024-11-16T05:51:30,113 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731736288173.9df9b8a2b43462b728e2352478a97347., pid=50, masterSystemTime=1731736289752 2024-11-16T05:51:30,114 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731736288173.ff61a3918e3c512081ba7b9c1e1a4948., pid=48, masterSystemTime=1731736289710 2024-11-16T05:51:30,116 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731736288173.9df9b8a2b43462b728e2352478a97347. 2024-11-16T05:51:30,116 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731736288173.9df9b8a2b43462b728e2352478a97347. 2024-11-16T05:51:30,116 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731736288173.de75e055385b7b1b204f0aa9c1924d0e. 2024-11-16T05:51:30,116 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => de75e055385b7b1b204f0aa9c1924d0e, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731736288173.de75e055385b7b1b204f0aa9c1924d0e.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-16T05:51:30,117 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true de75e055385b7b1b204f0aa9c1924d0e 2024-11-16T05:51:30,117 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731736288173.de75e055385b7b1b204f0aa9c1924d0e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:30,117 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for de75e055385b7b1b204f0aa9c1924d0e 2024-11-16T05:51:30,117 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for de75e055385b7b1b204f0aa9c1924d0e 2024-11-16T05:51:30,118 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731736288173.ff61a3918e3c512081ba7b9c1e1a4948. 2024-11-16T05:51:30,118 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731736288173.ff61a3918e3c512081ba7b9c1e1a4948. 2024-11-16T05:51:30,119 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=16 2024-11-16T05:51:30,119 INFO [StoreOpener-de75e055385b7b1b204f0aa9c1924d0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region de75e055385b7b1b204f0aa9c1924d0e 2024-11-16T05:51:30,120 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 408fc7812ce9cee6ea99428f155ce5a8, server=3456ee6a3164,43415,1731736285705 in 523 msec 2024-11-16T05:51:30,122 INFO [StoreOpener-de75e055385b7b1b204f0aa9c1924d0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de75e055385b7b1b204f0aa9c1924d0e columnFamilyName colfamily11 2024-11-16T05:51:30,122 DEBUG [StoreOpener-de75e055385b7b1b204f0aa9c1924d0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:30,123 INFO [StoreOpener-de75e055385b7b1b204f0aa9c1924d0e-1 {}] regionserver.HStore(327): Store=de75e055385b7b1b204f0aa9c1924d0e/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:30,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=408fc7812ce9cee6ea99428f155ce5a8, ASSIGN in 804 msec 2024-11-16T05:51:30,123 INFO [StoreOpener-de75e055385b7b1b204f0aa9c1924d0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region de75e055385b7b1b204f0aa9c1924d0e 2024-11-16T05:51:30,124 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=ff61a3918e3c512081ba7b9c1e1a4948, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,43415,1731736285705 2024-11-16T05:51:30,125 INFO [StoreOpener-de75e055385b7b1b204f0aa9c1924d0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de75e055385b7b1b204f0aa9c1924d0e columnFamilyName colfamily21 2024-11-16T05:51:30,125 DEBUG [StoreOpener-de75e055385b7b1b204f0aa9c1924d0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:30,126 INFO [StoreOpener-de75e055385b7b1b204f0aa9c1924d0e-1 {}] regionserver.HStore(327): Store=de75e055385b7b1b204f0aa9c1924d0e/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:30,126 INFO [StoreOpener-de75e055385b7b1b204f0aa9c1924d0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region de75e055385b7b1b204f0aa9c1924d0e 2024-11-16T05:51:30,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff61a3918e3c512081ba7b9c1e1a4948, server=3456ee6a3164,43415,1731736285705 because future has completed 2024-11-16T05:51:30,128 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=9df9b8a2b43462b728e2352478a97347, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,33007,1731736285465 2024-11-16T05:51:30,128 INFO [StoreOpener-de75e055385b7b1b204f0aa9c1924d0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de75e055385b7b1b204f0aa9c1924d0e columnFamilyName colfamily31 2024-11-16T05:51:30,129 DEBUG [StoreOpener-de75e055385b7b1b204f0aa9c1924d0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:30,130 INFO [StoreOpener-de75e055385b7b1b204f0aa9c1924d0e-1 {}] regionserver.HStore(327): Store=de75e055385b7b1b204f0aa9c1924d0e/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:30,130 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for de75e055385b7b1b204f0aa9c1924d0e 2024-11-16T05:51:30,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9df9b8a2b43462b728e2352478a97347, server=3456ee6a3164,33007,1731736285465 because future has completed 2024-11-16T05:51:30,132 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/de75e055385b7b1b204f0aa9c1924d0e 2024-11-16T05:51:30,132 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/de75e055385b7b1b204f0aa9c1924d0e 2024-11-16T05:51:30,134 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for de75e055385b7b1b204f0aa9c1924d0e 2024-11-16T05:51:30,134 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for de75e055385b7b1b204f0aa9c1924d0e 2024-11-16T05:51:30,135 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:51:30,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=12 2024-11-16T05:51:30,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure ff61a3918e3c512081ba7b9c1e1a4948, server=3456ee6a3164,43415,1731736285705 in 520 msec 2024-11-16T05:51:30,137 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for de75e055385b7b1b204f0aa9c1924d0e 2024-11-16T05:51:30,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ff61a3918e3c512081ba7b9c1e1a4948, ASSIGN in 820 msec 2024-11-16T05:51:30,140 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/de75e055385b7b1b204f0aa9c1924d0e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:51:30,141 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened de75e055385b7b1b204f0aa9c1924d0e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72621567, jitterRate=0.08214567601680756}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:51:30,141 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for de75e055385b7b1b204f0aa9c1924d0e 2024-11-16T05:51:30,141 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for de75e055385b7b1b204f0aa9c1924d0e: Running coprocessor pre-open hook at 1731736290117Writing region info on filesystem at 1731736290117Initializing all the Stores at 1731736290119 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736290119Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736290119Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736290119Cleaning up temporary data from old regions at 1731736290134 (+15 ms)Running coprocessor post-open hooks at 1731736290141 (+7 ms)Region opened successfully at 1731736290141 2024-11-16T05:51:30,142 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731736288173.de75e055385b7b1b204f0aa9c1924d0e., pid=53, masterSystemTime=1731736289752 2024-11-16T05:51:30,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=25 2024-11-16T05:51:30,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 9df9b8a2b43462b728e2352478a97347, server=3456ee6a3164,33007,1731736285465 in 517 msec 2024-11-16T05:51:30,145 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731736288173.de75e055385b7b1b204f0aa9c1924d0e. 2024-11-16T05:51:30,145 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731736288173.de75e055385b7b1b204f0aa9c1924d0e. 2024-11-16T05:51:30,145 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6. 2024-11-16T05:51:30,145 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => b59301af2f6042086ee9455ab281e7e6, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-16T05:51:30,146 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true b59301af2f6042086ee9455ab281e7e6 2024-11-16T05:51:30,146 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:51:30,146 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for b59301af2f6042086ee9455ab281e7e6 2024-11-16T05:51:30,146 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for b59301af2f6042086ee9455ab281e7e6 2024-11-16T05:51:30,146 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=de75e055385b7b1b204f0aa9c1924d0e, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,33007,1731736285465 2024-11-16T05:51:30,149 INFO [StoreOpener-b59301af2f6042086ee9455ab281e7e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region b59301af2f6042086ee9455ab281e7e6 2024-11-16T05:51:30,150 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure de75e055385b7b1b204f0aa9c1924d0e, server=3456ee6a3164,33007,1731736285465 because future has completed 2024-11-16T05:51:30,151 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9df9b8a2b43462b728e2352478a97347, ASSIGN in 833 msec 2024-11-16T05:51:30,152 INFO [StoreOpener-b59301af2f6042086ee9455ab281e7e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b59301af2f6042086ee9455ab281e7e6 columnFamilyName colfamily11 2024-11-16T05:51:30,152 DEBUG [StoreOpener-b59301af2f6042086ee9455ab281e7e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:30,153 INFO [StoreOpener-b59301af2f6042086ee9455ab281e7e6-1 {}] regionserver.HStore(327): Store=b59301af2f6042086ee9455ab281e7e6/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:30,153 INFO [StoreOpener-b59301af2f6042086ee9455ab281e7e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region b59301af2f6042086ee9455ab281e7e6 2024-11-16T05:51:30,155 INFO [StoreOpener-b59301af2f6042086ee9455ab281e7e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b59301af2f6042086ee9455ab281e7e6 columnFamilyName colfamily21 2024-11-16T05:51:30,155 DEBUG [StoreOpener-b59301af2f6042086ee9455ab281e7e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:30,155 INFO [StoreOpener-b59301af2f6042086ee9455ab281e7e6-1 {}] regionserver.HStore(327): Store=b59301af2f6042086ee9455ab281e7e6/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:30,156 INFO [StoreOpener-b59301af2f6042086ee9455ab281e7e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region b59301af2f6042086ee9455ab281e7e6 2024-11-16T05:51:30,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=7 2024-11-16T05:51:30,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure de75e055385b7b1b204f0aa9c1924d0e, server=3456ee6a3164,33007,1731736285465 in 516 msec 2024-11-16T05:51:30,158 INFO [StoreOpener-b59301af2f6042086ee9455ab281e7e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b59301af2f6042086ee9455ab281e7e6 columnFamilyName colfamily31 2024-11-16T05:51:30,158 DEBUG [StoreOpener-b59301af2f6042086ee9455ab281e7e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:51:30,158 INFO [StoreOpener-b59301af2f6042086ee9455ab281e7e6-1 {}] regionserver.HStore(327): Store=b59301af2f6042086ee9455ab281e7e6/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:51:30,159 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for b59301af2f6042086ee9455ab281e7e6 2024-11-16T05:51:30,159 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=de75e055385b7b1b204f0aa9c1924d0e, ASSIGN in 841 msec 2024-11-16T05:51:30,160 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b59301af2f6042086ee9455ab281e7e6 2024-11-16T05:51:30,161 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b59301af2f6042086ee9455ab281e7e6 2024-11-16T05:51:30,162 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for b59301af2f6042086ee9455ab281e7e6 2024-11-16T05:51:30,162 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for b59301af2f6042086ee9455ab281e7e6 2024-11-16T05:51:30,163 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:51:30,164 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=27 2024-11-16T05:51:30,164 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure eb73dbcec883aba378443bd58f1eb724, server=3456ee6a3164,33007,1731736285465 in 562 msec 2024-11-16T05:51:30,164 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for b59301af2f6042086ee9455ab281e7e6 2024-11-16T05:51:30,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=eb73dbcec883aba378443bd58f1eb724, ASSIGN in 848 msec 2024-11-16T05:51:30,167 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b59301af2f6042086ee9455ab281e7e6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T05:51:30,168 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened b59301af2f6042086ee9455ab281e7e6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70477013, jitterRate=0.05018933117389679}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:51:30,168 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b59301af2f6042086ee9455ab281e7e6 2024-11-16T05:51:30,168 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for b59301af2f6042086ee9455ab281e7e6: Running coprocessor pre-open hook at 1731736290146Writing region info on filesystem at 1731736290146Initializing all the Stores at 1731736290148 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736290148Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736290148Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736290148Cleaning up temporary data from old regions at 1731736290162 (+14 ms)Running coprocessor post-open hooks at 1731736290168 (+6 ms)Region opened successfully at 1731736290168 2024-11-16T05:51:30,169 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6., pid=52, masterSystemTime=1731736289752 2024-11-16T05:51:30,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=29 2024-11-16T05:51:30,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure afd4d1a884eb847b35b730a1b7a90e0d, server=3456ee6a3164,43415,1731736285705 in 590 msec 2024-11-16T05:51:30,173 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6. 2024-11-16T05:51:30,173 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6. 2024-11-16T05:51:30,175 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44685 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3456ee6a3164,33007,1731736285465, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3e30199c6045096f8f565c93af9b6088. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-16T05:51:30,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=afd4d1a884eb847b35b730a1b7a90e0d, ASSIGN in 857 msec 2024-11-16T05:51:30,177 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b59301af2f6042086ee9455ab281e7e6, regionState=OPEN, openSeqNum=2, regionLocation=3456ee6a3164,33007,1731736285465 2024-11-16T05:51:30,181 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b59301af2f6042086ee9455ab281e7e6, server=3456ee6a3164,33007,1731736285465 because future has completed 2024-11-16T05:51:30,182 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44685 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3456ee6a3164,43415,1731736285705, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=54bd8d95e5df76bdd2472b3a67cd2b8b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-16T05:51:30,187 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=5 2024-11-16T05:51:30,187 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure b59301af2f6042086ee9455ab281e7e6, server=3456ee6a3164,33007,1731736285465 in 551 msec 2024-11-16T05:51:30,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b59301af2f6042086ee9455ab281e7e6, ASSIGN in 871 msec 2024-11-16T05:51:30,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=6 2024-11-16T05:51:30,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 54bd8d95e5df76bdd2472b3a67cd2b8b, server=3456ee6a3164,43415,1731736285705 in 583 msec 2024-11-16T05:51:30,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=54bd8d95e5df76bdd2472b3a67cd2b8b, ASSIGN in 880 msec 2024-11-16T05:51:30,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=10 2024-11-16T05:51:30,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 3e30199c6045096f8f565c93af9b6088, server=3456ee6a3164,33007,1731736285465 in 559 msec 2024-11-16T05:51:30,205 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=4 2024-11-16T05:51:30,205 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=3e30199c6045096f8f565c93af9b6088, ASSIGN in 885 msec 2024-11-16T05:51:30,206 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T05:51:30,207 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731736290206"}]},"ts":"1731736290206"} 2024-11-16T05:51:30,209 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, state=ENABLED in hbase:meta 2024-11-16T05:51:30,211 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T05:51:30,217 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true in 2.0300 sec 2024-11-16T05:51:30,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44685 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T05:51:30,345 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true completed 2024-11-16T05:51:30,345 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true get assigned. Timeout = 60000ms 2024-11-16T05:51:30,347 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T05:51:30,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44781 {}] regionserver.StoreScanner(1138): Switch to stream read (scanned=32802 bytes) of info 2024-11-16T05:51:30,371 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true assigned to meta. Checking AM states. 2024-11-16T05:51:30,372 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T05:51:30,372 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true assigned. 2024-11-16T05:51:30,422 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aab', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:30,425 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aaa', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:30,428 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aad', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:30,429 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aae', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:30,430 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aaf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:30,431 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aac', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:30,432 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aag', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:30,433 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aah', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:30,478 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731736288173.54bd8d95e5df76bdd2472b3a67cd2b8b., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:30,480 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731736288173.54bd8d95e5df76bdd2472b3a67cd2b8b., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:30,482 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731736288173.54bd8d95e5df76bdd2472b3a67cd2b8b., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:30,484 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbe', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731736288173.54bd8d95e5df76bdd2472b3a67cd2b8b., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:30,485 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731736288173.54bd8d95e5df76bdd2472b3a67cd2b8b., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:30,487 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731736288173.54bd8d95e5df76bdd2472b3a67cd2b8b., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:30,530 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731736288173.de75e055385b7b1b204f0aa9c1924d0e., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:30,532 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731736288173.de75e055385b7b1b204f0aa9c1924d0e., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:30,534 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731736288173.de75e055385b7b1b204f0aa9c1924d0e., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:30,535 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='cce', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731736288173.de75e055385b7b1b204f0aa9c1924d0e., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:30,537 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731736288173.de75e055385b7b1b204f0aa9c1924d0e., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:30,538 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='cch', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731736288173.de75e055385b7b1b204f0aa9c1924d0e., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:30,577 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:30,579 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='dde', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:30,581 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:30,583 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:30,584 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:30,586 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:30,588 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:30,627 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eee', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731736288173.53faee5a89035a5faf9a235d93b486c9., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:30,629 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eef', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731736288173.53faee5a89035a5faf9a235d93b486c9., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:30,632 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eeg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731736288173.53faee5a89035a5faf9a235d93b486c9., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:30,633 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eeh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731736288173.53faee5a89035a5faf9a235d93b486c9., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:30,635 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eei', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731736288173.53faee5a89035a5faf9a235d93b486c9., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:30,637 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eej', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731736288173.53faee5a89035a5faf9a235d93b486c9., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:30,673 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='fff', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731736288173.3e30199c6045096f8f565c93af9b6088., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:30,676 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731736288173.3e30199c6045096f8f565c93af9b6088., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:30,679 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731736288173.3e30199c6045096f8f565c93af9b6088., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:30,681 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731736288173.3e30199c6045096f8f565c93af9b6088., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:30,683 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731736288173.3e30199c6045096f8f565c93af9b6088., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:30,684 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731736288173.3e30199c6045096f8f565c93af9b6088., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:30,721 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:30,723 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:30,725 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:30,727 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:30,729 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:30,731 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:30,733 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:30,767 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731736288173.ff61a3918e3c512081ba7b9c1e1a4948., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:30,770 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731736288173.ff61a3918e3c512081ba7b9c1e1a4948., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:30,772 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731736288173.ff61a3918e3c512081ba7b9c1e1a4948., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:30,773 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731736288173.ff61a3918e3c512081ba7b9c1e1a4948., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:30,775 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731736288173.ff61a3918e3c512081ba7b9c1e1a4948., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:30,777 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731736288173.ff61a3918e3c512081ba7b9c1e1a4948., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:30,822 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iij', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:30,825 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iik', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:30,827 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iil', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:30,829 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iim', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:30,832 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iin', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:30,834 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iio', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:30,835 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iii', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:30,837 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iip', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:30,893 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731736288173.5baf9d0bdd8c65c757ad796f93bef122., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:30,895 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731736288173.5baf9d0bdd8c65c757ad796f93bef122., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:30,899 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731736288173.5baf9d0bdd8c65c757ad796f93bef122., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:30,901 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731736288173.5baf9d0bdd8c65c757ad796f93bef122., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:30,903 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731736288173.5baf9d0bdd8c65c757ad796f93bef122., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:30,905 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731736288173.5baf9d0bdd8c65c757ad796f93bef122., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:30,958 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731736288173.63baab6432268e686bc2f01a98a6c75f., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:30,961 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731736288173.63baab6432268e686bc2f01a98a6c75f., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:30,964 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731736288173.63baab6432268e686bc2f01a98a6c75f., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:30,966 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731736288173.63baab6432268e686bc2f01a98a6c75f., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:30,969 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kko', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731736288173.63baab6432268e686bc2f01a98a6c75f., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:30,971 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731736288173.63baab6432268e686bc2f01a98a6c75f., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:31,005 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='lll', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731736288173.408fc7812ce9cee6ea99428f155ce5a8., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:31,008 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731736288173.408fc7812ce9cee6ea99428f155ce5a8., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:31,010 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='lln', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731736288173.408fc7812ce9cee6ea99428f155ce5a8., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:31,013 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731736288173.408fc7812ce9cee6ea99428f155ce5a8., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:31,015 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731736288173.408fc7812ce9cee6ea99428f155ce5a8., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:31,018 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731736288173.408fc7812ce9cee6ea99428f155ce5a8., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:31,057 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:31,060 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:31,062 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:31,065 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:31,067 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:31,070 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:31,103 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731736288173.12c6ab39e16284ff5b8ffeb36e672416., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:31,106 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nno', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731736288173.12c6ab39e16284ff5b8ffeb36e672416., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:31,108 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731736288173.12c6ab39e16284ff5b8ffeb36e672416., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:31,110 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731736288173.12c6ab39e16284ff5b8ffeb36e672416., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:31,112 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731736288173.12c6ab39e16284ff5b8ffeb36e672416., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:31,114 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nns', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731736288173.12c6ab39e16284ff5b8ffeb36e672416., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:31,155 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ooo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731736288173.07f1b4025bc2e2627b7dc3f7c79afd8e., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:31,158 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oop', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731736288173.07f1b4025bc2e2627b7dc3f7c79afd8e., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:31,160 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ooq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731736288173.07f1b4025bc2e2627b7dc3f7c79afd8e., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:31,162 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oor', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731736288173.07f1b4025bc2e2627b7dc3f7c79afd8e., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:31,164 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oot', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731736288173.07f1b4025bc2e2627b7dc3f7c79afd8e., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:31,166 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oos', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731736288173.07f1b4025bc2e2627b7dc3f7c79afd8e., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:31,204 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:31,206 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:31,208 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:31,210 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='pps', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:31,212 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:31,251 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731736288173.0166203610c1853876df2910d136e997., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:31,253 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731736288173.0166203610c1853876df2910d136e997., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:31,255 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqs', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731736288173.0166203610c1853876df2910d136e997., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:31,257 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731736288173.0166203610c1853876df2910d136e997., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:31,259 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731736288173.0166203610c1853876df2910d136e997., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:31,261 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731736288173.0166203610c1853876df2910d136e997., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:31,297 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731736288173.df4b7c945ebb852a560a10e20c40c386., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:31,299 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrs', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731736288173.df4b7c945ebb852a560a10e20c40c386., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:31,301 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731736288173.df4b7c945ebb852a560a10e20c40c386., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:31,303 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rru', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731736288173.df4b7c945ebb852a560a10e20c40c386., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:31,307 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731736288173.df4b7c945ebb852a560a10e20c40c386., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:31,310 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731736288173.df4b7c945ebb852a560a10e20c40c386., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:31,345 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='sss', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:31,348 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='sst', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:31,349 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:31,351 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:31,353 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:31,388 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731736288173.2beaf8caee4e552158f6bb091a0437bb., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:31,390 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731736288173.2beaf8caee4e552158f6bb091a0437bb., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:31,392 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731736288173.2beaf8caee4e552158f6bb091a0437bb., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:31,393 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731736288173.2beaf8caee4e552158f6bb091a0437bb., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:31,394 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731736288173.2beaf8caee4e552158f6bb091a0437bb., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:31,432 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731736288173.9df9b8a2b43462b728e2352478a97347., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:31,434 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731736288173.9df9b8a2b43462b728e2352478a97347., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:31,439 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731736288173.9df9b8a2b43462b728e2352478a97347., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:31,440 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uux', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731736288173.9df9b8a2b43462b728e2352478a97347., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:31,442 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731736288173.9df9b8a2b43462b728e2352478a97347., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:31,444 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731736288173.9df9b8a2b43462b728e2352478a97347., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:31,445 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uva', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731736288173.9df9b8a2b43462b728e2352478a97347., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:31,446 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uvb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731736288173.9df9b8a2b43462b728e2352478a97347., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:31,492 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:31,494 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:31,496 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:31,498 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:31,501 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:31,536 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='www', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731736288173.eb73dbcec883aba378443bd58f1eb724., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:31,540 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wwx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731736288173.eb73dbcec883aba378443bd58f1eb724., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:31,542 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wwy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731736288173.eb73dbcec883aba378443bd58f1eb724., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:31,544 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wwz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731736288173.eb73dbcec883aba378443bd58f1eb724., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:31,545 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wxa', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731736288173.eb73dbcec883aba378443bd58f1eb724., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:31,580 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:31,583 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:31,585 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:31,587 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xya', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:31,588 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xyb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:31,590 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xyc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db., hostname=3456ee6a3164,44781,1731736285630, seqNum=2] 2024-11-16T05:51:31,619 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yyy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731736288173.afd4d1a884eb847b35b730a1b7a90e0d., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:31,621 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yyz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731736288173.afd4d1a884eb847b35b730a1b7a90e0d., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:31,623 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yza', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731736288173.afd4d1a884eb847b35b730a1b7a90e0d., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:31,624 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yzb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731736288173.afd4d1a884eb847b35b730a1b7a90e0d., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:31,626 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yzc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731736288173.afd4d1a884eb847b35b730a1b7a90e0d., hostname=3456ee6a3164,43415,1731736285705, seqNum=2] 2024-11-16T05:51:31,672 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='zzz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731736288173.ca14297d921e6ca3fee0822ad2bb732e., hostname=3456ee6a3164,33007,1731736285465, seqNum=2] 2024-11-16T05:51:31,697 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:51:31,712 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:51:31,718 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38480, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:51:31,727 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53460, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:51:31,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33007 {}] regionserver.RSRpcServices(2610): Large batch operation detected (greater than 5000) (HBASE-18023). Requested Number of Rows: 5625 Client: jenkins//172.17.0.2 first region in multi=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6. 2024-11-16T05:51:31,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43415 {}] regionserver.RSRpcServices(2610): Large batch operation detected (greater than 5000) (HBASE-18023). Requested Number of Rows: 6327 Client: jenkins//172.17.0.2 first region in multi=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731736288173.54bd8d95e5df76bdd2472b3a67cd2b8b. 2024-11-16T05:51:31,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44781 {}] regionserver.RSRpcServices(2610): Large batch operation detected (greater than 5000) (HBASE-18023). Requested Number of Rows: 5624 Client: jenkins//172.17.0.2 first region in multi=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296. 2024-11-16T05:51:32,549 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,, stopping at row=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true ,, for max=2147483647 with caching=100 2024-11-16T05:51:32,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44781 {}] regionserver.StoreScanner(1138): Switch to stream read (scanned=32802 bytes) of info 2024-11-16T05:51:32,812 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: true 2024-11-16T05:51:32,812 INFO [Time-limited test {}] assignment.AssignmentTestingUtil(82): KILL REGION SERVER 3456ee6a3164,44781,1731736285630 2024-11-16T05:51:32,812 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 3456ee6a3164,44781,1731736285630 2024-11-16T05:51:32,813 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 3456ee6a3164,44781,1731736285630: Simulated kill ***** 2024-11-16T05:51:32,813 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-16T05:51:32,816 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-16T05:51:32,819 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-16T05:51:32,819 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-16T05:51:32,819 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-16T05:51:32,819 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 592129984 }, "NonHeapMemoryUsage": { "committed": 158007296, "init": 7667712, "max": -1, "used": 155109896 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [], "beans": [], "beans": [] } 2024-11-16T05:51:32,822 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44685 {}] master.MasterRpcServices(700): 3456ee6a3164,44781,1731736285630 reported a fatal error: ***** ABORTING region server 3456ee6a3164,44781,1731736285630: Simulated kill ***** 2024-11-16T05:51:32,825 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3456ee6a3164,44781,1731736285630' ***** 2024-11-16T05:51:32,825 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-16T05:51:32,826 INFO [RS:1;3456ee6a3164:44781 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T05:51:32,826 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T05:51:32,827 INFO [RS:1;3456ee6a3164:44781 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-16T05:51:32,827 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T05:51:32,827 INFO [RS:1;3456ee6a3164:44781 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-16T05:51:32,827 INFO [RS:1;3456ee6a3164:44781 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T05:51:32,827 INFO [RS:1;3456ee6a3164:44781 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;3456ee6a3164:44781. 2024-11-16T05:51:32,827 DEBUG [RS:1;3456ee6a3164:44781 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:51:32,832 DEBUG [RS:1;3456ee6a3164:44781 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:51:32,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741836_1012 (size=54214) 2024-11-16T05:51:32,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741836_1012 (size=54214) 2024-11-16T05:51:32,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741836_1012 (size=54214) 2024-11-16T05:51:32,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741835_1011 (size=297717) 2024-11-16T05:51:32,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741835_1011 (size=297717) 2024-11-16T05:51:32,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741835_1011 (size=297717) 2024-11-16T05:51:32,843 DEBUG [RS:1;3456ee6a3164:44781 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:51:32,843 INFO [RS:1;3456ee6a3164:44781 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T05:51:32,844 INFO [RS:1;3456ee6a3164:44781 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44781 2024-11-16T05:51:32,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44781-0x100471607020002, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3456ee6a3164,44781,1731736285630 2024-11-16T05:51:32,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T05:51:32,850 INFO [RS:1;3456ee6a3164:44781 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T05:51:32,852 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3456ee6a3164,44781,1731736285630] 2024-11-16T05:51:32,853 INFO [regionserver/3456ee6a3164:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T05:51:32,853 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3456ee6a3164,44781,1731736285630 already deleted, retry=false 2024-11-16T05:51:32,853 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 3456ee6a3164,44781,1731736285630 on 3456ee6a3164,44685,1731736284655 2024-11-16T05:51:32,858 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true 2024-11-16T05:51:32,862 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=57 for 3456ee6a3164,44781,1731736285630 (carryingMeta=true) 3456ee6a3164,44781,1731736285630/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@44c1e07e[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-16T05:51:32,863 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true 2024-11-16T05:51:32,863 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-16T05:51:32,863 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true 2024-11-16T05:51:32,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:51:32,863 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:51:32,927 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T05:51:32,930 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 0 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true 2024-11-16T05:51:32,931 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:51:32,931 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:51:32,931 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:51:32,931 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:51:32,932 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:51:32,932 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:51:32,932 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:51:32,932 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:51:32,933 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T05:51:32,941 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:51:32,942 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:51:32,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-16T05:51:32,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true 2024-11-16T05:51:32,945 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:51:32,945 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-16T05:51:32,947 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:51:32,947 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,44781,1731736285630, table=hbase:meta, region=1588230740 2024-11-16T05:51:32,948 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,44781,1731736285630, state=OPEN 2024-11-16T05:51:32,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:51:32,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:51:32,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:51:32,949 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:51:32,949 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:51:32,949 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:51:32,951 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736322951 2024-11-16T05:51:32,951 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:51:32,951 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true 2024-11-16T05:51:32,952 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-16T05:51:32,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44781-0x100471607020002, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:51:32,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44781-0x100471607020002, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:51:32,952 INFO [RS:1;3456ee6a3164:44781 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T05:51:32,952 INFO [RS:1;3456ee6a3164:44781 {}] regionserver.HRegionServer(1031): Exiting; stopping=3456ee6a3164,44781,1731736285630; zookeeper connection closed. 2024-11-16T05:51:32,953 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6b159082 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6b159082 2024-11-16T05:51:32,953 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3456ee6a3164:44781 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:51:32,955 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:51:32,955 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T05:51:32,955 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:51:32,956 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3456ee6a3164:44781 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:51:32,957 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true, isMeta: true 2024-11-16T05:51:32,959 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630-splitting 2024-11-16T05:51:32,961 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3456ee6a3164,44781,1731736285630 WAL count=1, meta=true 2024-11-16T05:51:32,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta}] 2024-11-16T05:51:32,963 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-16T05:51:32,963 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true 2024-11-16T05:51:32,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:51:32,963 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:51:33,064 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:51:33,064 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:51:33,065 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3456ee6a3164:44781 this server is in the failed servers list 2024-11-16T05:51:33,066 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:51:33,066 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:51:33,066 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:51:33,095 INFO [regionserver/3456ee6a3164:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-16T05:51:33,095 INFO [regionserver/3456ee6a3164:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-16T05:51:33,224 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T05:51:33,274 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:51:33,274 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:51:33,274 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3456ee6a3164:44781 this server is in the failed servers list 2024-11-16T05:51:33,275 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:51:33,275 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:51:33,275 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:51:33,277 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true' 2024-11-16T05:51:33,280 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-16T05:51:33,584 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:51:33,585 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:51:33,585 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3456ee6a3164:44781 this server is in the failed servers list 2024-11-16T05:51:33,586 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:51:33,586 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:51:33,586 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:51:34,094 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:51:34,094 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:51:34,095 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3456ee6a3164:44781 this server is in the failed servers list 2024-11-16T05:51:34,095 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:51:34,095 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:51:34,096 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:51:35,066 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-16T05:51:35,066 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-16T05:51:35,069 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-16T05:51:35,069 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true Metrics about Tables on a single HBase RegionServer 2024-11-16T05:51:35,071 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T05:51:35,071 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-16T05:51:35,071 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-16T05:51:35,071 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-16T05:51:35,104 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:51:35,104 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:51:35,105 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3456ee6a3164:44781 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:51:35,108 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,44781,1731736285630 for '' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 60000 ms, time elapsed = 2154 ms java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T05:51:35,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:51:35,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T05:51:35,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:51:35,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3456ee6a3164:44781 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:51:37,134 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:51:37,134 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:51:37,135 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3456ee6a3164:44781 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:51:37,136 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,44781,1731736285630 for '' in 1588230740 of hbase:meta failed, tries = 7, maxAttempts = 46, timeout = 60000 ms, time elapsed = 4184 ms java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T05:51:37,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:51:37,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T05:51:37,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:51:37,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3456ee6a3164:44781 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:51:41,174 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:51:41,174 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:51:41,175 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3456ee6a3164:44781 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:51:41,175 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,44781,1731736285630 for '' in 1588230740 of hbase:meta failed, tries = 8, maxAttempts = 46, timeout = 60000 ms, time elapsed = 8223 ms java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T05:51:41,176 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:51:41,176 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T05:51:41,176 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:51:41,176 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3456ee6a3164:44781 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:51:51,284 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:51:51,284 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:51:51,286 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3456ee6a3164:44781 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:51:51,287 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,44781,1731736285630 for '' in 1588230740 of hbase:meta failed, tries = 9, maxAttempts = 46, timeout = 60000 ms, time elapsed = 18335 ms java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T05:51:51,288 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:51:51,288 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T05:51:51,288 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:51:51,289 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3456ee6a3164:44781 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:51:53,729 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T05:51:56,779 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=3456ee6a3164:44685 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-16T05:52:01,344 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:52:01,344 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:52:01,346 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3456ee6a3164:44781 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:52:01,346 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,44781,1731736285630 for '' in 1588230740 of hbase:meta failed, tries = 10, maxAttempts = 46, timeout = 60000 ms, time elapsed = 28394 ms java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T05:52:01,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:52:01,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T05:52:01,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:52:01,348 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3456ee6a3164:44781 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:52:07,022 WARN [RS:0;3456ee6a3164:33007 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of 3456ee6a3164,44781,1731736285630: status=status unset, state=RUNNING, startTime=1731736292862, completionTime=-1 appears to have been leaked 2024-11-16T05:52:08,020 INFO [master/3456ee6a3164:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-16T05:52:08,020 INFO [master/3456ee6a3164:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-16T05:52:11,404 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:52:11,404 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:52:11,407 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3456ee6a3164:44781 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:52:11,409 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,44781,1731736285630 for '' in 1588230740 of hbase:meta failed, tries = 11, maxAttempts = 46, timeout = 60000 ms, time elapsed = 38457 ms java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T05:52:11,409 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:52:11,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T05:52:11,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:52:11,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3456ee6a3164:44781 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:52:21,474 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:52:21,474 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:52:21,476 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3456ee6a3164:44781 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:52:21,477 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,44781,1731736285630 for '' in 1588230740 of hbase:meta failed, tries = 12, maxAttempts = 46, timeout = 60000 ms, time elapsed = 48525 ms java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T05:52:21,478 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:52:21,478 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T05:52:21,478 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:52:21,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3456ee6a3164:44781 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:52:23,729 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T05:52:27,316 INFO [regionserver/3456ee6a3164:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: ExecutorStatusChore was stopped 2024-11-16T05:52:27,966 DEBUG [master/3456ee6a3164:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b59301af2f6042086ee9455ab281e7e6 changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:52:27,966 DEBUG [master/3456ee6a3164:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region de75e055385b7b1b204f0aa9c1924d0e changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:52:27,967 DEBUG [master/3456ee6a3164:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3e30199c6045096f8f565c93af9b6088 changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:52:27,968 DEBUG [master/3456ee6a3164:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 63baab6432268e686bc2f01a98a6c75f changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:52:27,968 DEBUG [master/3456ee6a3164:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 12c6ab39e16284ff5b8ffeb36e672416 changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:52:27,968 DEBUG [master/3456ee6a3164:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0166203610c1853876df2910d136e997 changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:52:27,968 DEBUG [master/3456ee6a3164:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9df9b8a2b43462b728e2352478a97347 changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:52:27,969 DEBUG [master/3456ee6a3164:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region eb73dbcec883aba378443bd58f1eb724 changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:52:27,969 DEBUG [master/3456ee6a3164:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ca14297d921e6ca3fee0822ad2bb732e changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:52:27,969 DEBUG [master/3456ee6a3164:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 54bd8d95e5df76bdd2472b3a67cd2b8b changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:52:27,970 DEBUG [master/3456ee6a3164:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 53faee5a89035a5faf9a235d93b486c9 changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:52:27,970 DEBUG [master/3456ee6a3164:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ff61a3918e3c512081ba7b9c1e1a4948 changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:52:27,970 DEBUG [master/3456ee6a3164:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5baf9d0bdd8c65c757ad796f93bef122 changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:52:27,970 DEBUG [master/3456ee6a3164:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 408fc7812ce9cee6ea99428f155ce5a8 changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:52:27,971 DEBUG [master/3456ee6a3164:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 07f1b4025bc2e2627b7dc3f7c79afd8e changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:52:27,971 DEBUG [master/3456ee6a3164:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region df4b7c945ebb852a560a10e20c40c386 changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:52:27,971 DEBUG [master/3456ee6a3164:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2beaf8caee4e552158f6bb091a0437bb changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:52:27,971 DEBUG [master/3456ee6a3164:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region afd4d1a884eb847b35b730a1b7a90e0d changed from -1.0 to 0.0, refreshing cache 2024-11-16T05:52:31,783 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=3456ee6a3164:44685 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-16T05:52:32,956 WARN [Time-limited test {}] procedure.MasterProcedureTestingUtility$3(130): Failed to load meta org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=13, exceptions: 2024-11-16T05:51:32.956Z, java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:51:33.067Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:51:33.275Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:51:33.586Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:51:34.096Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:51:35.110Z, java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:51:37.137Z, java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:51:41.176Z, java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:51:51.288Z, java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:52:01.347Z, java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:52:11.410Z, java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:52:21.478Z, java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:493) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:461) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:422) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.fullScanRegions(MetaTableAccessor.java:108) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionStateStore.visitMeta(RegionStateStore.java:108) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.loadMeta(AssignmentManager.java:1907) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.joinCluster(AssignmentManager.java:1769) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:121) ~[test-classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:143) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:477) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:486) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPBase.testRecoveryAndDoubleExecution(TestSCPBase.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta.testRecoveryAndDoubleExecutionOnRsWithMeta(TestSCPWithMeta.java:36) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:102) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-16T05:52:32,962 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 1 exec state=pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true 2024-11-16T05:52:32,963 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:52:32,963 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:52:32,963 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:52:32,963 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:52:32,964 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:52:32,964 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:52:32,964 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:52:32,965 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:52:32,967 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-16T05:52:32,970 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:52:32,970 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:52:32,971 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-16T05:52:32,971 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=false; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true 2024-11-16T05:52:32,971 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=false; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:52:32,971 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-16T05:52:32,972 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:52:32,972 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,44781,1731736285630, table=hbase:meta, region=1588230740 2024-11-16T05:52:32,973 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,44781,1731736285630, state=OPEN 2024-11-16T05:52:32,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:52:32,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:52:32,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:52:32,975 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:52:32,975 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:52:32,975 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:52:32,976 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736382976 2024-11-16T05:52:32,976 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:52:32,976 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true, isMeta: true 2024-11-16T05:52:32,977 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:52:32,977 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:52:32,978 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3456ee6a3164:44781 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:52:32,978 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:52:32,978 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T05:52:32,978 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:52:32,978 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3456ee6a3164,44781,1731736285630 WAL count=1, meta=true 2024-11-16T05:52:32,978 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3456ee6a3164:44781 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:52:32,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta}] 2024-11-16T05:52:32,978 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-16T05:52:32,983 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=3456ee6a3164,43415,1731736285705 2024-11-16T05:52:32,985 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705}] 2024-11-16T05:52:33,084 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:52:33,084 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:52:33,084 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3456ee6a3164:44781 this server is in the failed servers list 2024-11-16T05:52:33,085 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:52:33,085 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:52:33,085 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:52:33,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43415 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-11-16T05:52:33,174 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630-splitting/3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, size=52.9 K (54214bytes) 2024-11-16T05:52:33,175 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-16T05:52:33,175 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-16T05:52:33,175 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630-splitting/3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta 2024-11-16T05:52:33,180 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630-splitting/3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta after 3ms 2024-11-16T05:52:33,188 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630-splitting/3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta took 15ms 2024-11-16T05:52:33,199 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 2024-11-16T05:52:33,207 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta.temp 2024-11-16T05:52:33,210 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630-splitting/3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta so closing down 2024-11-16T05:52:33,210 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T05:52:33,211 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta.temp 2024-11-16T05:52:33,216 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T05:52:33,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741864_1040 (size=53973) 2024-11-16T05:52:33,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741864_1040 (size=53973) 2024-11-16T05:52:33,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741864_1040 (size=53973) 2024-11-16T05:52:33,228 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta.temp (wrote 57 edits, skipped 0 edits in 3 ms) 2024-11-16T05:52:33,230 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta.temp to hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-16T05:52:33,234 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 41 ms; skipped=1; WAL=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630-splitting/3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, size=52.9 K, length=54214, corrupted=false, cancelled=false 2024-11-16T05:52:33,234 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630-splitting/3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, journal: Splitting hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630-splitting/3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, size=52.9 K (54214bytes) at 1731736353174Finishing writing output for hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630-splitting/3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta so closing down at 1731736353210 (+36 ms)Creating recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta.temp at 1731736353211 (+1 ms)3 split writer threads finished at 1731736353216 (+5 ms)Closed recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta.temp (wrote 57 edits, skipped 0 edits in 3 ms) at 1731736353228 (+12 ms)Rename recovered edits hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta.temp to hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1731736353230 (+2 ms)Processed 58 edits across 1 Regions in 41 ms; skipped=1; WAL=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630-splitting/3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, size=52.9 K, length=54214, corrupted=false, cancelled=false at 1731736353234 (+4 ms) 2024-11-16T05:52:33,235 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630-splitting/3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta 2024-11-16T05:52:33,236 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-16T05:52:33,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44685 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-16T05:52:33,246 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630-splitting/3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta to hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/oldWALs 2024-11-16T05:52:33,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-16T05:52:33,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705 in 261 msec 2024-11-16T05:52:33,251 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=3456ee6a3164,43415,1731736285705 2024-11-16T05:52:33,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-16T05:52:33,255 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705 in 274 msec 2024-11-16T05:52:33,257 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630-splitting dir is empty, no logs to split. 2024-11-16T05:52:33,257 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3456ee6a3164,44781,1731736285630 WAL count=0, meta=true 2024-11-16T05:52:33,257 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 3456ee6a3164,44781,1731736285630 WAL splitting is done? wals=0, meta=true 2024-11-16T05:52:33,257 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-16T05:52:33,257 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true 2024-11-16T05:52:33,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:52:33,258 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:52:33,294 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:52:33,294 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:52:33,294 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3456ee6a3164:44781 this server is in the failed servers list 2024-11-16T05:52:33,295 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:52:33,295 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:52:33,295 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:52:33,603 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:52:33,604 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:52:33,604 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3456ee6a3164:44781 this server is in the failed servers list 2024-11-16T05:52:33,604 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:52:33,604 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:52:33,604 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:52:34,113 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:52:34,114 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:52:34,114 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3456ee6a3164:44781 this server is in the failed servers list 2024-11-16T05:52:34,114 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:52:34,114 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:52:34,114 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:52:35,124 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:52:35,124 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:52:35,125 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3456ee6a3164:44781 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:52:35,125 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,44781,1731736285630 for '' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 60000 ms, time elapsed = 2149 ms java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T05:52:35,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:52:35,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T05:52:35,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:52:35,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3456ee6a3164:44781 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:52:37,144 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:52:37,145 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:52:37,147 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3456ee6a3164:44781 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:52:37,148 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,44781,1731736285630 for '' in 1588230740 of hbase:meta failed, tries = 7, maxAttempts = 46, timeout = 60000 ms, time elapsed = 4171 ms java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T05:52:37,150 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:52:37,150 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T05:52:37,150 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:52:37,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3456ee6a3164:44781 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:52:41,184 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:52:41,184 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:52:41,186 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3456ee6a3164:44781 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:52:41,186 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,44781,1731736285630 for '' in 1588230740 of hbase:meta failed, tries = 8, maxAttempts = 46, timeout = 60000 ms, time elapsed = 8209 ms java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T05:52:41,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:52:41,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T05:52:41,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:52:41,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3456ee6a3164:44781 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:52:51,224 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:52:51,224 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:52:51,225 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3456ee6a3164:44781 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:52:51,226 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,44781,1731736285630 for '' in 1588230740 of hbase:meta failed, tries = 9, maxAttempts = 46, timeout = 60000 ms, time elapsed = 18249 ms java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T05:52:51,226 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:52:51,226 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T05:52:51,226 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:52:51,226 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3456ee6a3164:44781 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:52:53,730 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T05:53:01,314 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:53:01,314 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:53:01,316 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3456ee6a3164:44781 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:53:01,317 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,44781,1731736285630 for '' in 1588230740 of hbase:meta failed, tries = 10, maxAttempts = 46, timeout = 60000 ms, time elapsed = 28340 ms java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T05:53:01,319 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:53:01,319 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T05:53:01,319 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:53:01,320 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3456ee6a3164:44781 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:53:11,394 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:53:11,394 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:53:11,395 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3456ee6a3164:44781 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:53:11,396 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,44781,1731736285630 for '' in 1588230740 of hbase:meta failed, tries = 11, maxAttempts = 46, timeout = 60000 ms, time elapsed = 38419 ms java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T05:53:11,397 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:53:11,397 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T05:53:11,397 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:53:11,397 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3456ee6a3164:44781 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:53:21,444 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:53:21,444 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:53:21,446 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3456ee6a3164:44781 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:53:21,447 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,44781,1731736285630 for '' in 1588230740 of hbase:meta failed, tries = 12, maxAttempts = 46, timeout = 60000 ms, time elapsed = 48470 ms java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T05:53:21,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:53:21,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T05:53:21,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:53:21,449 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3456ee6a3164:44781 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:53:23,730 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T05:53:27,025 WARN [regionserver/3456ee6a3164:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 20, running: 1 2024-11-16T05:53:27,025 WARN [regionserver/3456ee6a3164:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 17, running: 1 2024-11-16T05:53:30,667 INFO [regionserver/3456ee6a3164:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactedHFilesCleaner was stopped 2024-11-16T05:53:32,985 WARN [Time-limited test {}] procedure.MasterProcedureTestingUtility$3(130): Failed to load meta org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=13, exceptions: 2024-11-16T05:52:32.978Z, java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:52:33.085Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:52:33.295Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:52:33.604Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:52:34.114Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:52:35.126Z, java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:52:37.150Z, java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:52:41.187Z, java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:52:51.226Z, java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:53:01.319Z, java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:53:11.397Z, java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:53:21.448Z, java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:493) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:461) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:422) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.fullScanRegions(MetaTableAccessor.java:108) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionStateStore.visitMeta(RegionStateStore.java:108) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.loadMeta(AssignmentManager.java:1907) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.joinCluster(AssignmentManager.java:1769) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:121) ~[test-classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:143) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:477) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:486) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPBase.testRecoveryAndDoubleExecution(TestSCPBase.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta.testRecoveryAndDoubleExecutionOnRsWithMeta(TestSCPWithMeta.java:36) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:102) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-16T05:53:32,990 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 2 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true 2024-11-16T05:53:32,990 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:53:32,991 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:53:32,991 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:53:32,991 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:53:32,992 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:53:32,992 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:53:32,993 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:53:32,994 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:53:32,995 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-16T05:53:32,998 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:53:32,998 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:53:32,998 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:53:32,998 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:53:32,999 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-16T05:53:33,000 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44685 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:53:33,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=false; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true 2024-11-16T05:53:33,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705 2024-11-16T05:53:33,000 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705 2024-11-16T05:53:33,000 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=false; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:53:33,000 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:53:33,000 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:53:33,001 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-16T05:53:33,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:53:33,002 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,44781,1731736285630, table=hbase:meta, region=1588230740 2024-11-16T05:53:33,002 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,44781,1731736285630, state=OPEN 2024-11-16T05:53:33,002 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44685 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:53:33,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:53:33,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:53:33,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:53:33,004 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:53:33,004 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:53:33,004 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:53:33,005 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736443005 2024-11-16T05:53:33,005 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:53:33,006 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:53:33,006 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:53:33,007 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3456ee6a3164:44781 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:53:33,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:53:33,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T05:53:33,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:53:33,008 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630-splitting dir is empty, no logs to split. 2024-11-16T05:53:33,008 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3456ee6a3164,44781,1731736285630 WAL count=0, meta=true 2024-11-16T05:53:33,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3456ee6a3164:44781 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:53:33,008 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 3456ee6a3164,44781,1731736285630 WAL splitting is done? wals=0, meta=true 2024-11-16T05:53:33,008 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-16T05:53:33,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T05:53:33,010 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-16T05:53:33,011 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_GET_REGIONS, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true 2024-11-16T05:53:33,011 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:53:33,011 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_GET_REGIONS, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:53:33,114 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:53:33,114 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:53:33,115 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3456ee6a3164:44781 this server is in the failed servers list 2024-11-16T05:53:33,115 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:53:33,115 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:53:33,115 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:53:33,324 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:53:33,325 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:53:33,325 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3456ee6a3164:44781 this server is in the failed servers list 2024-11-16T05:53:33,326 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:53:33,326 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:53:33,326 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:53:33,634 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:53:33,634 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:53:33,634 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3456ee6a3164:44781 this server is in the failed servers list 2024-11-16T05:53:33,634 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:53:33,634 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:53:33,635 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:53:34,144 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:53:34,144 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:53:34,145 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3456ee6a3164:44781 this server is in the failed servers list 2024-11-16T05:53:34,145 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:53:34,145 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:53:34,146 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:53:35,154 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:53:35,154 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:53:35,155 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3456ee6a3164:44781 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:53:35,155 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,44781,1731736285630 for '' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 60000 ms, time elapsed = 2149 ms java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T05:53:35,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:53:35,156 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T05:53:35,156 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:53:35,156 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3456ee6a3164:44781 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:53:37,164 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:53:37,164 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:53:37,167 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3456ee6a3164:44781 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:53:37,168 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,44781,1731736285630 for '' in 1588230740 of hbase:meta failed, tries = 7, maxAttempts = 46, timeout = 60000 ms, time elapsed = 4161 ms java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T05:53:37,169 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:53:37,169 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T05:53:37,169 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:53:37,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3456ee6a3164:44781 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:53:37,786 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=3456ee6a3164:44685 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-16T05:53:41,184 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:53:41,184 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:53:41,186 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3456ee6a3164:44781 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:53:41,188 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,44781,1731736285630 for '' in 1588230740 of hbase:meta failed, tries = 8, maxAttempts = 46, timeout = 60000 ms, time elapsed = 8181 ms java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T05:53:41,189 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:53:41,189 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T05:53:41,189 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:53:41,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3456ee6a3164:44781 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:53:51,264 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:53:51,264 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:53:51,265 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3456ee6a3164:44781 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:53:51,265 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,44781,1731736285630 for '' in 1588230740 of hbase:meta failed, tries = 9, maxAttempts = 46, timeout = 60000 ms, time elapsed = 18259 ms java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T05:53:51,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:53:51,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T05:53:51,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:53:51,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3456ee6a3164:44781 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:53:53,731 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T05:54:01,324 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:54:01,324 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:54:01,325 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3456ee6a3164:44781 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:54:01,325 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,44781,1731736285630 for '' in 1588230740 of hbase:meta failed, tries = 10, maxAttempts = 46, timeout = 60000 ms, time elapsed = 28319 ms java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T05:54:01,326 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:54:01,326 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T05:54:01,326 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:54:01,326 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3456ee6a3164:44781 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:54:07,668 WARN [RS:2;3456ee6a3164:43415 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of 3456ee6a3164,44781,1731736285630: status=Processing ServerCrashProcedure of 3456ee6a3164,44781,1731736285630 current State SERVER_CRASH_SPLIT_META_LOGS, state=RUNNING, startTime=1731736292944, completionTime=-1 appears to have been leaked 2024-11-16T05:54:07,668 WARN [RS:2;3456ee6a3164:43415 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of 3456ee6a3164,44781,1731736285630: status=Processing ServerCrashProcedure of 3456ee6a3164,44781,1731736285630 current State SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, state=RUNNING, startTime=1731736352971, completionTime=-1 appears to have been leaked 2024-11-16T05:54:11,384 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:54:11,384 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:54:11,385 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3456ee6a3164:44781 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:54:11,386 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,44781,1731736285630 for '' in 1588230740 of hbase:meta failed, tries = 11, maxAttempts = 46, timeout = 60000 ms, time elapsed = 38379 ms java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T05:54:11,386 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:54:11,386 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T05:54:11,386 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:54:11,386 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3456ee6a3164:44781 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:54:21,434 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:54:21,435 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:54:21,436 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3456ee6a3164:44781 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:54:21,438 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3456ee6a3164,44781,1731736285630 for '' in 1588230740 of hbase:meta failed, tries = 12, maxAttempts = 46, timeout = 60000 ms, time elapsed = 48431 ms java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-16T05:54:21,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:54:21,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T05:54:21,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:54:21,440 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3456ee6a3164:44781 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:54:23,731 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T05:54:33,015 WARN [Time-limited test {}] procedure.MasterProcedureTestingUtility$3(130): Failed to load meta org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=13, exceptions: 2024-11-16T05:53:33.008Z, java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:53:33.115Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:53:33.326Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:53:33.635Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:53:34.146Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:53:35.156Z, java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:53:37.169Z, java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:53:41.189Z, java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:53:51.266Z, java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:54:01.326Z, java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:54:11.386Z, java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:54:21.439Z, java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:493) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:461) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:422) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.fullScanRegions(MetaTableAccessor.java:108) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionStateStore.visitMeta(RegionStateStore.java:108) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.loadMeta(AssignmentManager.java:1907) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.joinCluster(AssignmentManager.java:1769) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:121) ~[test-classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:143) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:477) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:486) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPBase.testRecoveryAndDoubleExecution(TestSCPBase.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta.testRecoveryAndDoubleExecutionOnRsWithMeta(TestSCPWithMeta.java:36) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:102) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-16T05:54:33,017 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 3 exec state=pid=57, state=WAITING:SERVER_CRASH_GET_REGIONS, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true 2024-11-16T05:54:33,017 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:54:33,017 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:33,017 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:33,017 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:33,018 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:54:33,018 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:54:33,018 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:54:33,018 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:54:33,019 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-16T05:54:33,023 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:54:33,023 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:33,023 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:33,023 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:33,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-16T05:54:33,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=false; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true 2024-11-16T05:54:33,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:33,025 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:33,025 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=false; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:33,025 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:33,025 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:33,026 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-16T05:54:33,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:33,026 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,44781,1731736285630, table=hbase:meta, region=1588230740 2024-11-16T05:54:33,026 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,44781,1731736285630, state=OPEN 2024-11-16T05:54:33,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:33,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:33,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:33,028 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:33,028 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:33,028 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:33,029 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736503029 2024-11-16T05:54:33,029 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:54:33,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T05:54:33,029 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-16T05:54:33,029 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:54:33,029 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:54:33,030 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3456ee6a3164:44781 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:54:33,031 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:54:33,031 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T05:54:33,031 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:54:33,031 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3456ee6a3164:44781 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:54:33,032 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:33,033 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T05:54:33,134 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:54:33,134 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1] 2024-11-16T05:54:33,134 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3456ee6a3164:44781 this server is in the failed servers list 2024-11-16T05:54:33,135 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:54:33,135 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:54:33,135 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:54:33,183 DEBUG [3456ee6a3164:44685 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-16T05:54:33,184 DEBUG [3456ee6a3164:44685 {}] balancer.BalancerClusterState(204): Hosts are {3456ee6a3164=0} racks are {/default-rack=0} 2024-11-16T05:54:33,184 DEBUG [3456ee6a3164:44685 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T05:54:33,184 DEBUG [3456ee6a3164:44685 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T05:54:33,184 DEBUG [3456ee6a3164:44685 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T05:54:33,184 DEBUG [3456ee6a3164:44685 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T05:54:33,184 INFO [3456ee6a3164:44685 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T05:54:33,184 INFO [3456ee6a3164:44685 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T05:54:33,184 DEBUG [3456ee6a3164:44685 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T05:54:33,184 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:33,185 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,43415,1731736285705, state=OPENING 2024-11-16T05:54:33,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:33,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:33,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:33,188 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:33,188 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:33,188 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:33,188 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:33,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,43415,1731736285705}] 2024-11-16T05:54:33,343 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38007, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T05:54:33,344 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:54:33,344 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,43415,1731736285705, seqNum=-1] 2024-11-16T05:54:33,344 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T05:54:33,346 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53749, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:54:33,349 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T05:54:33,349 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T05:54:33,349 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T05:54:33,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43415 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Scan size: 121 connection: 172.17.0.2:53749 deadline: 1731736533032, exception=org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on 3456ee6a3164,43415,1731736285705 2024-11-16T05:54:33,352 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,43415,1731736285705, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,43415,1731736285705, seqNum=-1, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on 3456ee6a3164,43415,1731736285705 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:54:33,352 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3456ee6a3164%2C43415%2C1731736285705.meta, suffix=.meta, logDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,43415,1731736285705, archiveDir=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/oldWALs, maxLogs=32 2024-11-16T05:54:33,352 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,43415,1731736285705, seqNum=-1 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: hbase:meta,,1 is not online on 3456ee6a3164,43415,1731736285705 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:54:33,352 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,43415,1731736285705, seqNum=-1 from cache 2024-11-16T05:54:33,375 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,43415,1731736285705/3456ee6a3164%2C43415%2C1731736285705.meta.1731736473355.meta, exclude list is [], retry=0 2024-11-16T05:54:33,378 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33441,DS-f0a76934-b7f3-4998-a613-d00266c6901a,DISK] 2024-11-16T05:54:33,378 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32873,DS-56cc9d96-a128-48c4-add0-03db81c0fd8d,DISK] 2024-11-16T05:54:33,378 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36993,DS-d9cb5fed-ca91-420f-8b4c-9e853c8c3426,DISK] 2024-11-16T05:54:33,387 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,43415,1731736285705/3456ee6a3164%2C43415%2C1731736285705.meta.1731736473355.meta 2024-11-16T05:54:33,388 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37155:37155),(127.0.0.1/127.0.0.1:37513:37513),(127.0.0.1/127.0.0.1:43727:43727)] 2024-11-16T05:54:33,388 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T05:54:33,389 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T05:54:33,389 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T05:54:33,389 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T05:54:33,389 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T05:54:33,389 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:33,390 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T05:54:33,390 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T05:54:33,396 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T05:54:33,398 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T05:54:33,398 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:33,398 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:54:33,398 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T05:54:33,400 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T05:54:33,400 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:33,401 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:54:33,401 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T05:54:33,402 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T05:54:33,402 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:33,403 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:54:33,403 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T05:54:33,404 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T05:54:33,404 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:33,405 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T05:54:33,405 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T05:54:33,406 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740 2024-11-16T05:54:33,408 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740 2024-11-16T05:54:33,410 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-16T05:54:33,429 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-16T05:54:33,434 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=49.58 KB heapSize=69.98 KB 2024-11-16T05:54:33,534 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/.tmp/info/f1cc884e45254de78ab6140f7a8891c2 is 268, key is testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731736288173.54bd8d95e5df76bdd2472b3a67cd2b8b./info:regioninfo/1731736289917/Put/seqid=0 2024-11-16T05:54:33,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741866_1042 (size=58053) 2024-11-16T05:54:33,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741866_1042 (size=58053) 2024-11-16T05:54:33,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741866_1042 (size=58053) 2024-11-16T05:54:33,549 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.31 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/.tmp/info/f1cc884e45254de78ab6140f7a8891c2 2024-11-16T05:54:33,654 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/.tmp/ns/e4ff9858cc83402a9bb0dc55d4fc0171 is 43, key is default/ns:d/1731736287886/Put/seqid=0 2024-11-16T05:54:33,664 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:54:33,664 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,43415,1731736285705, seqNum=-1] 2024-11-16T05:54:33,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43415 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 121 connection: 172.17.0.2:53749 deadline: 1731736533029, exception=org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 3456ee6a3164,43415,1731736285705 2024-11-16T05:54:33,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,43415,1731736285705, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,43415,1731736285705, seqNum=-1, error=org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 3456ee6a3164,43415,1731736285705 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:54:33,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,43415,1731736285705, seqNum=-1 is org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 3456ee6a3164,43415,1731736285705 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T05:54:33,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,43415,1731736285705, seqNum=-1 from cache 2024-11-16T05:54:33,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741867_1043 (size=5153) 2024-11-16T05:54:33,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741867_1043 (size=5153) 2024-11-16T05:54:33,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741867_1043 (size=5153) 2024-11-16T05:54:33,723 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/.tmp/ns/e4ff9858cc83402a9bb0dc55d4fc0171 2024-11-16T05:54:33,760 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/.tmp/table/6e4036e34d7342e791b89cd71884d277 is 105, key is testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/table:state/1731736290206/Put/seqid=0 2024-11-16T05:54:33,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741868_1044 (size=5620) 2024-11-16T05:54:33,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741868_1044 (size=5620) 2024-11-16T05:54:33,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741868_1044 (size=5620) 2024-11-16T05:54:33,773 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=202 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/.tmp/table/6e4036e34d7342e791b89cd71884d277 2024-11-16T05:54:33,787 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/.tmp/info/f1cc884e45254de78ab6140f7a8891c2 as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/info/f1cc884e45254de78ab6140f7a8891c2 2024-11-16T05:54:33,798 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/info/f1cc884e45254de78ab6140f7a8891c2, entries=260, sequenceid=60, filesize=56.7 K 2024-11-16T05:54:33,801 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/.tmp/ns/e4ff9858cc83402a9bb0dc55d4fc0171 as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/ns/e4ff9858cc83402a9bb0dc55d4fc0171 2024-11-16T05:54:33,809 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/ns/e4ff9858cc83402a9bb0dc55d4fc0171, entries=2, sequenceid=60, filesize=5.0 K 2024-11-16T05:54:33,811 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/.tmp/table/6e4036e34d7342e791b89cd71884d277 as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/table/6e4036e34d7342e791b89cd71884d277 2024-11-16T05:54:33,820 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/table/6e4036e34d7342e791b89cd71884d277, entries=2, sequenceid=60, filesize=5.5 K 2024-11-16T05:54:33,825 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~49.58 KB/50772, heapSize ~69.69 KB/71360, currentSize=0 B/0 for 1588230740 in 391ms, sequenceid=60, compaction requested=false; wal=null 2024-11-16T05:54:33,827 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-16T05:54:33,828 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T05:54:33,828 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T05:54:33,829 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T05:54:33,831 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T05:54:33,834 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-16T05:54:33,836 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74657144, jitterRate=0.11247813701629639}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T05:54:33,836 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T05:54:33,836 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731736473390Writing region info on filesystem at 1731736473390Initializing all the Stores at 1731736473392 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736473392Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736473396 (+4 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736473396Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731736473396Obtaining lock to block concurrent updates at 1731736473434 (+38 ms)Preparing flush snapshotting stores in 1588230740 at 1731736473434Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=50772, getHeapSize=71600, getOffHeapSize=0, getCellsCount=264 at 1731736473439 (+5 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731736473439Flushing 1588230740/info: creating writer at 1731736473443 (+4 ms)Flushing 1588230740/info: appending metadata at 1731736473521 (+78 ms)Flushing 1588230740/info: closing flushed file at 1731736473525 (+4 ms)Flushing 1588230740/ns: creating writer at 1731736473625 (+100 ms)Flushing 1588230740/ns: appending metadata at 1731736473653 (+28 ms)Flushing 1588230740/ns: closing flushed file at 1731736473653Flushing 1588230740/table: creating writer at 1731736473736 (+83 ms)Flushing 1588230740/table: appending metadata at 1731736473758 (+22 ms)Flushing 1588230740/table: closing flushed file at 1731736473758Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7988412d: reopening flushed file at 1731736473785 (+27 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b8aaadc: reopening flushed file at 1731736473799 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@358b3b0b: reopening flushed file at 1731736473810 (+11 ms)Finished flush of dataSize ~49.58 KB/50772, heapSize ~69.69 KB/71360, currentSize=0 B/0 for 1588230740 in 391ms, sequenceid=60, compaction requested=false; wal=null at 1731736473825 (+15 ms)Cleaning up temporary data from old regions at 1731736473828 (+3 ms)Running coprocessor post-open hooks at 1731736473836 (+8 ms)Region opened successfully at 1731736473836 2024-11-16T05:54:33,838 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1731736473341 2024-11-16T05:54:33,841 DEBUG [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T05:54:33,841 INFO [RS_OPEN_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T05:54:33,842 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:33,843 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,43415,1731736285705, state=OPEN 2024-11-16T05:54:33,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:33,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:33,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:33,845 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:33,845 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:33,845 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:33,845 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:33,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-16T05:54:33,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,43415,1731736285705 in 657 msec 2024-11-16T05:54:33,851 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-11-16T05:54:33,851 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 819 msec 2024-11-16T05:54:34,174 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:54:34,174 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,43415,1731736285705, seqNum=-1] 2024-11-16T05:54:34,207 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b59301af2f6042086ee9455ab281e7e6, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:34,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54bd8d95e5df76bdd2472b3a67cd2b8b, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:34,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de75e055385b7b1b204f0aa9c1924d0e, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:34,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42e678de61063128349f238929e3d296, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:34,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53faee5a89035a5faf9a235d93b486c9, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:34,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3e30199c6045096f8f565c93af9b6088, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:34,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=883c672d1e7de6e08e15c0c42278e66d, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:34,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff61a3918e3c512081ba7b9c1e1a4948, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:34,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c81e1c1af980b616d932866cdd4ff915, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:34,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5baf9d0bdd8c65c757ad796f93bef122, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:34,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=63baab6432268e686bc2f01a98a6c75f, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:34,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=408fc7812ce9cee6ea99428f155ce5a8, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:34,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd79de8195c0452a334d7da789e8f23b, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:34,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12c6ab39e16284ff5b8ffeb36e672416, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:34,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07f1b4025bc2e2627b7dc3f7c79afd8e, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:34,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec037e054803eda72bad2d81c6143b00, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:34,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0166203610c1853876df2910d136e997, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:34,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df4b7c945ebb852a560a10e20c40c386, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:34,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6c8ea5676b855dac5a939ace6afadb0, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:34,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2beaf8caee4e552158f6bb091a0437bb, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:34,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9df9b8a2b43462b728e2352478a97347, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:34,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a3e438f5882d261cd799e4d6b75095, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:34,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb73dbcec883aba378443bd58f1eb724, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:34,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac83f7461cc15e741d0440c6ee2781db, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:34,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afd4d1a884eb847b35b730a1b7a90e0d, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:34,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca14297d921e6ca3fee0822ad2bb732e, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:34,211 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:54:34,211 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736534211 2024-11-16T05:54:34,211 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736594211 2024-11-16T05:54:34,211 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 1.1820 sec 2024-11-16T05:54:34,212 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 3456ee6a3164,44781,1731736285630 had 9 regions 2024-11-16T05:54:34,212 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-16T05:54:34,212 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true 2024-11-16T05:54:34,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:34,212 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:34,462 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 4 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true 2024-11-16T05:54:34,462 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:54:34,462 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:34,462 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:34,462 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:34,462 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:54:34,462 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:54:34,462 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:54:34,463 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:54:34,464 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T05:54:34,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:54:34,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:34,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:34,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:34,467 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:34,467 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:34,468 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-16T05:54:34,469 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_GET_REGIONS, hasLock=false; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true 2024-11-16T05:54:34,469 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:34,469 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:34,469 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:34,469 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:34,470 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_GET_REGIONS, hasLock=false; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:34,470 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:34,470 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:34,470 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:34,470 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,43415,1731736285705 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:34,470 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-16T05:54:34,471 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=61 2024-11-16T05:54:34,471 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,43415,1731736285705, table=hbase:meta, region=1588230740 2024-11-16T05:54:34,471 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,43415,1731736285705, state=OPEN 2024-11-16T05:54:34,489 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44685 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:34,510 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44685 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:34,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:34,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:34,521 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:34,521 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:34,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:34,521 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:34,522 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736504522 2024-11-16T05:54:34,522 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:54:34,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b59301af2f6042086ee9455ab281e7e6, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:34,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54bd8d95e5df76bdd2472b3a67cd2b8b, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:34,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de75e055385b7b1b204f0aa9c1924d0e, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:34,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42e678de61063128349f238929e3d296, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:34,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53faee5a89035a5faf9a235d93b486c9, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:34,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3e30199c6045096f8f565c93af9b6088, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:34,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=883c672d1e7de6e08e15c0c42278e66d, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:34,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff61a3918e3c512081ba7b9c1e1a4948, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:34,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c81e1c1af980b616d932866cdd4ff915, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:34,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5baf9d0bdd8c65c757ad796f93bef122, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:34,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=63baab6432268e686bc2f01a98a6c75f, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:34,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=408fc7812ce9cee6ea99428f155ce5a8, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:34,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd79de8195c0452a334d7da789e8f23b, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:34,533 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12c6ab39e16284ff5b8ffeb36e672416, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:34,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07f1b4025bc2e2627b7dc3f7c79afd8e, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:34,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec037e054803eda72bad2d81c6143b00, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:34,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0166203610c1853876df2910d136e997, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:34,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df4b7c945ebb852a560a10e20c40c386, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:34,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6c8ea5676b855dac5a939ace6afadb0, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:34,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2beaf8caee4e552158f6bb091a0437bb, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:34,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9df9b8a2b43462b728e2352478a97347, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:34,534 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a3e438f5882d261cd799e4d6b75095, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:34,535 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb73dbcec883aba378443bd58f1eb724, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:34,535 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac83f7461cc15e741d0440c6ee2781db, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:34,535 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afd4d1a884eb847b35b730a1b7a90e0d, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:34,535 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca14297d921e6ca3fee0822ad2bb732e, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:34,535 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:54:34,535 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736534535 2024-11-16T05:54:34,535 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736594535 2024-11-16T05:54:34,535 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-16T05:54:34,535 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 3456ee6a3164,44781,1731736285630 had 8 regions 2024-11-16T05:54:34,536 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-16T05:54:34,537 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true, isMeta: false 2024-11-16T05:54:34,539 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3456ee6a3164,44781,1731736285630 WAL count=1, meta=false 2024-11-16T05:54:34,539 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 3456ee6a3164%2C44781%2C1731736285630.1731736287193}] 2024-11-16T05:54:34,539 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-16T05:54:34,539 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true 2024-11-16T05:54:34,539 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:34,540 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:34,785 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 5 exec state=pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true 2024-11-16T05:54:34,786 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:54:34,786 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:34,786 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:34,786 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:34,787 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:54:34,787 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:54:34,787 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:54:34,788 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:54:34,789 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-16T05:54:34,792 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:54:34,792 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:34,792 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:34,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:34,793 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:34,793 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:34,793 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-16T05:54:34,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=false; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true 2024-11-16T05:54:34,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:34,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:34,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:34,794 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:34,794 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=false; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:34,794 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:34,794 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:34,794 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:34,794 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,43415,1731736285705 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:34,794 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-16T05:54:34,795 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=61 2024-11-16T05:54:34,795 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,43415,1731736285705, table=hbase:meta, region=1588230740 2024-11-16T05:54:34,795 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,43415,1731736285705, state=OPEN 2024-11-16T05:54:34,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:34,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:34,797 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:34,797 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:34,797 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:34,797 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:34,798 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736504798 2024-11-16T05:54:34,799 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:54:34,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b59301af2f6042086ee9455ab281e7e6, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:34,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54bd8d95e5df76bdd2472b3a67cd2b8b, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:34,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de75e055385b7b1b204f0aa9c1924d0e, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:34,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42e678de61063128349f238929e3d296, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:34,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53faee5a89035a5faf9a235d93b486c9, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:34,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3e30199c6045096f8f565c93af9b6088, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:34,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=883c672d1e7de6e08e15c0c42278e66d, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:34,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff61a3918e3c512081ba7b9c1e1a4948, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:34,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c81e1c1af980b616d932866cdd4ff915, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:34,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5baf9d0bdd8c65c757ad796f93bef122, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:34,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=63baab6432268e686bc2f01a98a6c75f, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:34,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=408fc7812ce9cee6ea99428f155ce5a8, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:34,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd79de8195c0452a334d7da789e8f23b, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:34,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12c6ab39e16284ff5b8ffeb36e672416, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:34,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07f1b4025bc2e2627b7dc3f7c79afd8e, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:34,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec037e054803eda72bad2d81c6143b00, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:34,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0166203610c1853876df2910d136e997, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:34,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df4b7c945ebb852a560a10e20c40c386, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:34,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6c8ea5676b855dac5a939ace6afadb0, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:34,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2beaf8caee4e552158f6bb091a0437bb, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:34,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9df9b8a2b43462b728e2352478a97347, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:34,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a3e438f5882d261cd799e4d6b75095, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:34,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb73dbcec883aba378443bd58f1eb724, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:34,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac83f7461cc15e741d0440c6ee2781db, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:34,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afd4d1a884eb847b35b730a1b7a90e0d, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:34,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca14297d921e6ca3fee0822ad2bb732e, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:34,809 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:54:34,809 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736534809 2024-11-16T05:54:34,809 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736594809 2024-11-16T05:54:34,809 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-16T05:54:34,810 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true, isMeta: false 2024-11-16T05:54:34,812 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3456ee6a3164,44781,1731736285630 WAL count=1, meta=false 2024-11-16T05:54:34,812 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 3456ee6a3164%2C44781%2C1731736285630.1731736287193}] 2024-11-16T05:54:34,812 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-16T05:54:34,815 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=3456ee6a3164,33007,1731736285465 2024-11-16T05:54:34,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C44781%2C1731736285630.1731736287193, worker=3456ee6a3164,33007,1731736285465}] 2024-11-16T05:54:34,971 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56173, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T05:54:34,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33007 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-11-16T05:54:34,990 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630-splitting/3456ee6a3164%2C44781%2C1731736285630.1731736287193, size=290.7 K (297717bytes) 2024-11-16T05:54:34,990 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630-splitting/3456ee6a3164%2C44781%2C1731736285630.1731736287193 2024-11-16T05:54:34,990 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630-splitting/3456ee6a3164%2C44781%2C1731736285630.1731736287193 after 0ms 2024-11-16T05:54:34,994 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630-splitting/3456ee6a3164%2C44781%2C1731736285630.1731736287193 took 5ms 2024-11-16T05:54:34,999 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 10a3e438f5882d261cd799e4d6b75095: last_flushed_sequence_id: 18446744073709551615 2024-11-16T05:54:35,000 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 42e678de61063128349f238929e3d296: last_flushed_sequence_id: 18446744073709551615 2024-11-16T05:54:35,002 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for ac83f7461cc15e741d0440c6ee2781db: last_flushed_sequence_id: 18446744073709551615 2024-11-16T05:54:35,003 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for c81e1c1af980b616d932866cdd4ff915: last_flushed_sequence_id: 18446744073709551615 2024-11-16T05:54:35,005 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for a6c8ea5676b855dac5a939ace6afadb0: last_flushed_sequence_id: 18446744073709551615 2024-11-16T05:54:35,006 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 883c672d1e7de6e08e15c0c42278e66d: last_flushed_sequence_id: 18446744073709551615 2024-11-16T05:54:35,007 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for bd79de8195c0452a334d7da789e8f23b: last_flushed_sequence_id: 18446744073709551615 2024-11-16T05:54:35,009 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for ec037e054803eda72bad2d81c6143b00: last_flushed_sequence_id: 18446744073709551615 2024-11-16T05:54:35,013 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp 2024-11-16T05:54:35,014 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp 2024-11-16T05:54:35,015 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/42e678de61063128349f238929e3d296/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp 2024-11-16T05:54:35,016 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/883c672d1e7de6e08e15c0c42278e66d/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp 2024-11-16T05:54:35,017 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp 2024-11-16T05:54:35,019 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c81e1c1af980b616d932866cdd4ff915/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp 2024-11-16T05:54:35,019 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp 2024-11-16T05:54:35,021 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bd79de8195c0452a334d7da789e8f23b/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp 2024-11-16T05:54:35,021 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp 2024-11-16T05:54:35,023 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ec037e054803eda72bad2d81c6143b00/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp 2024-11-16T05:54:35,026 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp 2024-11-16T05:54:35,027 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630-splitting/3456ee6a3164%2C44781%2C1731736285630.1731736287193 so closing down 2024-11-16T05:54:35,027 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a6c8ea5676b855dac5a939ace6afadb0/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp 2024-11-16T05:54:35,027 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-16T05:54:35,029 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp 2024-11-16T05:54:35,030 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp 2024-11-16T05:54:35,036 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10a3e438f5882d261cd799e4d6b75095/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp 2024-11-16T05:54:35,036 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac83f7461cc15e741d0440c6ee2781db/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp 2024-11-16T05:54:35,038 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-11-16T05:54:35,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741869_1045 (size=36767) 2024-11-16T05:54:35,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741869_1045 (size=36767) 2024-11-16T05:54:35,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741869_1045 (size=36767) 2024-11-16T05:54:35,053 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c81e1c1af980b616d932866cdd4ff915/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-16T05:54:35,054 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c81e1c1af980b616d932866cdd4ff915/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp to hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c81e1c1af980b616d932866cdd4ff915/recovered.edits/0000000000000000004 2024-11-16T05:54:35,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741870_1046 (size=36767) 2024-11-16T05:54:35,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741870_1046 (size=36767) 2024-11-16T05:54:35,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741870_1046 (size=36767) 2024-11-16T05:54:35,068 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a6c8ea5676b855dac5a939ace6afadb0/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-16T05:54:35,070 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a6c8ea5676b855dac5a939ace6afadb0/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp to hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a6c8ea5676b855dac5a939ace6afadb0/recovered.edits/0000000000000000004 2024-11-16T05:54:35,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741871_1047 (size=36767) 2024-11-16T05:54:35,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741871_1047 (size=36767) 2024-11-16T05:54:35,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741871_1047 (size=36767) 2024-11-16T05:54:35,085 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10a3e438f5882d261cd799e4d6b75095/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp (wrote 1 edits, skipped 0 edits in 1 ms) 2024-11-16T05:54:35,088 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10a3e438f5882d261cd799e4d6b75095/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp to hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10a3e438f5882d261cd799e4d6b75095/recovered.edits/0000000000000000004 2024-11-16T05:54:35,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741872_1048 (size=36767) 2024-11-16T05:54:35,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741872_1048 (size=36767) 2024-11-16T05:54:35,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741872_1048 (size=36767) 2024-11-16T05:54:35,095 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/42e678de61063128349f238929e3d296/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp (wrote 1 edits, skipped 0 edits in 2 ms) 2024-11-16T05:54:35,097 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/42e678de61063128349f238929e3d296/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp to hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/42e678de61063128349f238929e3d296/recovered.edits/0000000000000000004 2024-11-16T05:54:35,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741873_1049 (size=36767) 2024-11-16T05:54:35,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741873_1049 (size=36767) 2024-11-16T05:54:35,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741873_1049 (size=36767) 2024-11-16T05:54:35,107 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/883c672d1e7de6e08e15c0c42278e66d/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp (wrote 1 edits, skipped 0 edits in 1 ms) 2024-11-16T05:54:35,109 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/883c672d1e7de6e08e15c0c42278e66d/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp to hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/883c672d1e7de6e08e15c0c42278e66d/recovered.edits/0000000000000000004 2024-11-16T05:54:35,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741874_1050 (size=36767) 2024-11-16T05:54:35,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741874_1050 (size=36767) 2024-11-16T05:54:35,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741874_1050 (size=36767) 2024-11-16T05:54:35,120 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bd79de8195c0452a334d7da789e8f23b/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-16T05:54:35,122 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bd79de8195c0452a334d7da789e8f23b/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp to hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bd79de8195c0452a334d7da789e8f23b/recovered.edits/0000000000000000004 2024-11-16T05:54:35,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741875_1051 (size=36767) 2024-11-16T05:54:35,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741875_1051 (size=36767) 2024-11-16T05:54:35,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741875_1051 (size=36767) 2024-11-16T05:54:35,129 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ec037e054803eda72bad2d81c6143b00/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-16T05:54:35,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741876_1052 (size=36767) 2024-11-16T05:54:35,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741876_1052 (size=36767) 2024-11-16T05:54:35,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741876_1052 (size=36767) 2024-11-16T05:54:35,131 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ec037e054803eda72bad2d81c6143b00/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp to hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ec037e054803eda72bad2d81c6143b00/recovered.edits/0000000000000000004 2024-11-16T05:54:35,531 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac83f7461cc15e741d0440c6ee2781db/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp (wrote 1 edits, skipped 0 edits in 1 ms) 2024-11-16T05:54:35,535 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac83f7461cc15e741d0440c6ee2781db/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp to hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac83f7461cc15e741d0440c6ee2781db/recovered.edits/0000000000000000004 2024-11-16T05:54:35,535 INFO [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 16 edits across 8 Regions in 539 ms; skipped=8; WAL=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630-splitting/3456ee6a3164%2C44781%2C1731736285630.1731736287193, size=290.7 K, length=297717, corrupted=false, cancelled=false 2024-11-16T05:54:35,536 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630-splitting/3456ee6a3164%2C44781%2C1731736285630.1731736287193, journal: Splitting hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630-splitting/3456ee6a3164%2C44781%2C1731736285630.1731736287193, size=290.7 K (297717bytes) at 1731736474990Creating recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/42e678de61063128349f238929e3d296/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp at 1731736475015 (+25 ms)Creating recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/883c672d1e7de6e08e15c0c42278e66d/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp at 1731736475016 (+1 ms)Creating recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c81e1c1af980b616d932866cdd4ff915/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp at 1731736475019 (+3 ms)Creating recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bd79de8195c0452a334d7da789e8f23b/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp at 1731736475021 (+2 ms)Creating recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ec037e054803eda72bad2d81c6143b00/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp at 1731736475023 (+2 ms)Split 6 edits, skipped 8 edits. at 1731736475024 (+1 ms)Finishing writing output for hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630-splitting/3456ee6a3164%2C44781%2C1731736285630.1731736287193 so closing down at 1731736475027 (+3 ms)Creating recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a6c8ea5676b855dac5a939ace6afadb0/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp at 1731736475027Creating recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10a3e438f5882d261cd799e4d6b75095/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp at 1731736475036 (+9 ms)Creating recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10a3e438f5882d261cd799e4d6b75095/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp at 17317364750363 split writer threads finished at 1731736475038 (+2 ms)Closed recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c81e1c1af980b616d932866cdd4ff915/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731736475053 (+15 ms)Rename recovered edits hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c81e1c1af980b616d932866cdd4ff915/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp to hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c81e1c1af980b616d932866cdd4ff915/recovered.edits/0000000000000000004 at 1731736475054 (+1 ms)Closed recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a6c8ea5676b855dac5a939ace6afadb0/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731736475068 (+14 ms)Rename recovered edits hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a6c8ea5676b855dac5a939ace6afadb0/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp to hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a6c8ea5676b855dac5a939ace6afadb0/recovered.edits/0000000000000000004 at 1731736475070 (+2 ms)Closed recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10a3e438f5882d261cd799e4d6b75095/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp (wrote 1 edits, skipped 0 edits in 1 ms) at 1731736475086 (+16 ms)Rename recovered edits hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10a3e438f5882d261cd799e4d6b75095/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp to hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10a3e438f5882d261cd799e4d6b75095/recovered.edits/0000000000000000004 at 1731736475088 (+2 ms)Closed recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/42e678de61063128349f238929e3d296/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp (wrote 1 edits, skipped 0 edits in 2 ms) at 1731736475095 (+7 ms)Rename recovered edits hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/42e678de61063128349f238929e3d296/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp to hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/42e678de61063128349f238929e3d296/recovered.edits/0000000000000000004 at 1731736475097 (+2 ms)Closed recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/883c672d1e7de6e08e15c0c42278e66d/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp (wrote 1 edits, skipped 0 edits in 1 ms) at 1731736475107 (+10 ms)Rename recovered edits hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/883c672d1e7de6e08e15c0c42278e66d/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp to hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/883c672d1e7de6e08e15c0c42278e66d/recovered.edits/0000000000000000004 at 1731736475109 (+2 ms)Closed recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bd79de8195c0452a334d7da789e8f23b/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731736475120 (+11 ms)Rename recovered edits hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bd79de8195c0452a334d7da789e8f23b/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp to hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bd79de8195c0452a334d7da789e8f23b/recovered.edits/0000000000000000004 at 1731736475122 (+2 ms)Closed recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ec037e054803eda72bad2d81c6143b00/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731736475129 (+7 ms)Rename recovered edits hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ec037e054803eda72bad2d81c6143b00/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp to hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ec037e054803eda72bad2d81c6143b00/recovered.edits/0000000000000000004 at 1731736475131 (+2 ms)Closed recovered edits writer path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac83f7461cc15e741d0440c6ee2781db/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp (wrote 1 edits, skipped 0 edits in 1 ms) at 1731736475531 (+400 ms)Rename recovered edits hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac83f7461cc15e741d0440c6ee2781db/recovered.edits/0000000000000000004-3456ee6a3164%2C44781%2C1731736285630.1731736287193.temp to hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac83f7461cc15e741d0440c6ee2781db/recovered.edits/0000000000000000004 at 1731736475535 (+4 ms)Processed 16 edits across 8 Regions in 539 ms; skipped=8; WAL=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630-splitting/3456ee6a3164%2C44781%2C1731736285630.1731736287193, size=290.7 K, length=297717, corrupted=false, cancelled=false at 1731736475535 2024-11-16T05:54:35,536 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630-splitting/3456ee6a3164%2C44781%2C1731736285630.1731736287193 2024-11-16T05:54:35,536 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3456ee6a3164:0-0 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-16T05:54:35,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44685 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-16T05:54:35,542 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630-splitting/3456ee6a3164%2C44781%2C1731736285630.1731736287193 to hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/oldWALs 2024-11-16T05:54:35,544 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-16T05:54:35,544 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C44781%2C1731736285630.1731736287193, worker=3456ee6a3164,33007,1731736285465 in 726 msec 2024-11-16T05:54:35,545 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=3456ee6a3164,33007,1731736285465 2024-11-16T05:54:35,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-16T05:54:35,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C44781%2C1731736285630.1731736287193, worker=3456ee6a3164,33007,1731736285465 in 733 msec 2024-11-16T05:54:35,549 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630-splitting dir is empty, no logs to split. 2024-11-16T05:54:35,549 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3456ee6a3164,44781,1731736285630 WAL count=0, meta=false 2024-11-16T05:54:35,549 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 3456ee6a3164,44781,1731736285630 WAL splitting is done? wals=0, meta=false 2024-11-16T05:54:35,550 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-16T05:54:35,550 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true 2024-11-16T05:54:35,550 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:35,550 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:35,560 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 6 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true 2024-11-16T05:54:35,560 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:54:35,560 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:35,561 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:35,561 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:35,561 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:54:35,561 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:54:35,561 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:54:35,561 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:54:35,562 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T05:54:35,564 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:54:35,564 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:35,564 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:35,564 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:35,564 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:35,564 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:35,564 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:35,565 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:35,565 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-16T05:54:35,566 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=false; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true 2024-11-16T05:54:35,566 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:35,566 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:35,566 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:35,566 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:35,566 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C44781%2C1731736285630.1731736287193, worker=3456ee6a3164,33007,1731736285465 2024-11-16T05:54:35,566 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C44781%2C1731736285630.1731736287193, worker=3456ee6a3164,33007,1731736285465 2024-11-16T05:54:35,567 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=false; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:35,567 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:35,567 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:35,567 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:35,567 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,43415,1731736285705 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:35,567 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C44781%2C1731736285630.1731736287193, worker=3456ee6a3164,33007,1731736285465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:35,567 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C44781%2C1731736285630.1731736287193, worker=3456ee6a3164,33007,1731736285465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:35,567 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-16T05:54:35,568 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=61 2024-11-16T05:54:35,568 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,43415,1731736285705, table=hbase:meta, region=1588230740 2024-11-16T05:54:35,568 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,43415,1731736285705, state=OPEN 2024-11-16T05:54:35,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:35,571 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:35,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:35,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:35,572 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:35,572 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:35,572 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736505572 2024-11-16T05:54:35,572 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:54:35,577 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b59301af2f6042086ee9455ab281e7e6, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:35,577 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54bd8d95e5df76bdd2472b3a67cd2b8b, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:35,577 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de75e055385b7b1b204f0aa9c1924d0e, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:35,577 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42e678de61063128349f238929e3d296, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:35,578 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53faee5a89035a5faf9a235d93b486c9, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:35,578 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3e30199c6045096f8f565c93af9b6088, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:35,578 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=883c672d1e7de6e08e15c0c42278e66d, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:35,578 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff61a3918e3c512081ba7b9c1e1a4948, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:35,578 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c81e1c1af980b616d932866cdd4ff915, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:35,578 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5baf9d0bdd8c65c757ad796f93bef122, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:35,578 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=63baab6432268e686bc2f01a98a6c75f, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:35,578 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=408fc7812ce9cee6ea99428f155ce5a8, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:35,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd79de8195c0452a334d7da789e8f23b, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:35,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12c6ab39e16284ff5b8ffeb36e672416, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:35,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07f1b4025bc2e2627b7dc3f7c79afd8e, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:35,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec037e054803eda72bad2d81c6143b00, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:35,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0166203610c1853876df2910d136e997, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:35,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df4b7c945ebb852a560a10e20c40c386, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:35,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6c8ea5676b855dac5a939ace6afadb0, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:35,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2beaf8caee4e552158f6bb091a0437bb, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:35,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9df9b8a2b43462b728e2352478a97347, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:35,579 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a3e438f5882d261cd799e4d6b75095, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:35,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb73dbcec883aba378443bd58f1eb724, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:35,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac83f7461cc15e741d0440c6ee2781db, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:35,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afd4d1a884eb847b35b730a1b7a90e0d, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:35,580 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca14297d921e6ca3fee0822ad2bb732e, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:35,580 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:54:35,580 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736535580 2024-11-16T05:54:35,580 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736595580 2024-11-16T05:54:35,580 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-16T05:54:35,582 INFO [PEWorker-1 {}] master.MasterWalManager(331): Log dir for server 3456ee6a3164,44781,1731736285630 does not exist 2024-11-16T05:54:35,582 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3456ee6a3164,44781,1731736285630 WAL count=0, meta=false 2024-11-16T05:54:35,582 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 3456ee6a3164,44781,1731736285630 WAL splitting is done? wals=0, meta=false 2024-11-16T05:54:35,583 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 3456ee6a3164,44781,1731736285630 failed, ignore...File hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/WALs/3456ee6a3164,44781,1731736285630-splitting does not exist. 2024-11-16T05:54:35,583 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-16T05:54:35,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c81e1c1af980b616d932866cdd4ff915, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=883c672d1e7de6e08e15c0c42278e66d, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ac83f7461cc15e741d0440c6ee2781db, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ec037e054803eda72bad2d81c6143b00, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=42e678de61063128349f238929e3d296, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bd79de8195c0452a334d7da789e8f23b, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a6c8ea5676b855dac5a939ace6afadb0, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=10a3e438f5882d261cd799e4d6b75095, ASSIGN}] 2024-11-16T05:54:35,585 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-16T05:54:35,585 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_FINISH, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true 2024-11-16T05:54:35,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:35,585 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_FINISH, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:35,830 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 7 exec state=pid=57, state=WAITING:SERVER_CRASH_FINISH, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true 2024-11-16T05:54:35,830 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:54:35,831 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:35,831 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:35,831 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:35,831 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:54:35,831 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:54:35,831 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:54:35,832 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:54:35,832 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T05:54:35,834 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:54:35,834 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:35,834 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:35,834 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:35,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:35,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:35,835 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:35,835 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:35,836 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-16T05:54:35,837 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=false; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true 2024-11-16T05:54:35,838 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:35,838 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:35,838 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:35,838 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:35,838 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C44781%2C1731736285630.1731736287193, worker=3456ee6a3164,33007,1731736285465 2024-11-16T05:54:35,838 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C44781%2C1731736285630.1731736287193, worker=3456ee6a3164,33007,1731736285465 2024-11-16T05:54:35,838 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=false; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:35,838 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:35,838 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:35,838 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:35,838 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,43415,1731736285705 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:35,838 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C44781%2C1731736285630.1731736287193, worker=3456ee6a3164,33007,1731736285465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:35,838 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C44781%2C1731736285630.1731736287193, worker=3456ee6a3164,33007,1731736285465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:35,838 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-16T05:54:35,839 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=61 2024-11-16T05:54:35,839 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,43415,1731736285705, table=hbase:meta, region=1588230740 2024-11-16T05:54:35,839 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,43415,1731736285705, state=OPEN 2024-11-16T05:54:35,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:35,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:35,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:35,841 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:35,841 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:35,841 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:35,842 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736505842 2024-11-16T05:54:35,842 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:54:35,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b59301af2f6042086ee9455ab281e7e6, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:35,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54bd8d95e5df76bdd2472b3a67cd2b8b, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:35,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de75e055385b7b1b204f0aa9c1924d0e, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:35,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42e678de61063128349f238929e3d296, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:35,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53faee5a89035a5faf9a235d93b486c9, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:35,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3e30199c6045096f8f565c93af9b6088, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:35,846 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=883c672d1e7de6e08e15c0c42278e66d, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:35,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff61a3918e3c512081ba7b9c1e1a4948, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:35,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c81e1c1af980b616d932866cdd4ff915, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:35,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5baf9d0bdd8c65c757ad796f93bef122, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:35,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=63baab6432268e686bc2f01a98a6c75f, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:35,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=408fc7812ce9cee6ea99428f155ce5a8, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:35,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd79de8195c0452a334d7da789e8f23b, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:35,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12c6ab39e16284ff5b8ffeb36e672416, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:35,847 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07f1b4025bc2e2627b7dc3f7c79afd8e, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:35,848 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec037e054803eda72bad2d81c6143b00, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:35,848 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0166203610c1853876df2910d136e997, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:35,848 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df4b7c945ebb852a560a10e20c40c386, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:35,848 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6c8ea5676b855dac5a939ace6afadb0, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:35,848 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2beaf8caee4e552158f6bb091a0437bb, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:35,848 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9df9b8a2b43462b728e2352478a97347, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:35,848 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a3e438f5882d261cd799e4d6b75095, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:35,849 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb73dbcec883aba378443bd58f1eb724, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:35,849 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac83f7461cc15e741d0440c6ee2781db, regionState=OPEN, lastHost=3456ee6a3164,44781,1731736285630, regionLocation=3456ee6a3164,44781,1731736285630, openSeqNum=2 2024-11-16T05:54:35,849 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afd4d1a884eb847b35b730a1b7a90e0d, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:35,849 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca14297d921e6ca3fee0822ad2bb732e, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:35,849 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:54:35,849 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736535849 2024-11-16T05:54:35,849 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736595849 2024-11-16T05:54:35,849 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-16T05:54:35,850 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c81e1c1af980b616d932866cdd4ff915, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=883c672d1e7de6e08e15c0c42278e66d, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ac83f7461cc15e741d0440c6ee2781db, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ec037e054803eda72bad2d81c6143b00, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=42e678de61063128349f238929e3d296, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bd79de8195c0452a334d7da789e8f23b, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a6c8ea5676b855dac5a939ace6afadb0, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=10a3e438f5882d261cd799e4d6b75095, ASSIGN}] 2024-11-16T05:54:35,850 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-16T05:54:35,851 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=10a3e438f5882d261cd799e4d6b75095, ASSIGN 2024-11-16T05:54:35,852 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=10a3e438f5882d261cd799e4d6b75095, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T05:54:35,853 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a6c8ea5676b855dac5a939ace6afadb0, ASSIGN 2024-11-16T05:54:35,854 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a6c8ea5676b855dac5a939ace6afadb0, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T05:54:35,854 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bd79de8195c0452a334d7da789e8f23b, ASSIGN 2024-11-16T05:54:35,855 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bd79de8195c0452a334d7da789e8f23b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T05:54:35,856 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=42e678de61063128349f238929e3d296, ASSIGN 2024-11-16T05:54:35,856 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=42e678de61063128349f238929e3d296, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T05:54:35,857 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ec037e054803eda72bad2d81c6143b00, ASSIGN 2024-11-16T05:54:35,858 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ec037e054803eda72bad2d81c6143b00, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T05:54:35,858 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ac83f7461cc15e741d0440c6ee2781db, ASSIGN 2024-11-16T05:54:35,859 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ac83f7461cc15e741d0440c6ee2781db, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T05:54:35,859 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=883c672d1e7de6e08e15c0c42278e66d, ASSIGN 2024-11-16T05:54:35,860 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=883c672d1e7de6e08e15c0c42278e66d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T05:54:35,861 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c81e1c1af980b616d932866cdd4ff915, ASSIGN 2024-11-16T05:54:35,861 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c81e1c1af980b616d932866cdd4ff915, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T05:54:36,003 DEBUG [3456ee6a3164:44685 {}] balancer.BalancerClusterState(204): Hosts are {3456ee6a3164=0} racks are {/default-rack=0} 2024-11-16T05:54:36,003 DEBUG [3456ee6a3164:44685 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T05:54:36,003 DEBUG [3456ee6a3164:44685 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T05:54:36,003 DEBUG [3456ee6a3164:44685 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T05:54:36,003 DEBUG [3456ee6a3164:44685 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T05:54:36,003 INFO [3456ee6a3164:44685 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T05:54:36,003 INFO [3456ee6a3164:44685 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T05:54:36,003 DEBUG [3456ee6a3164:44685 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T05:54:36,004 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=10a3e438f5882d261cd799e4d6b75095, regionState=OPENING, regionLocation=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:36,004 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=ec037e054803eda72bad2d81c6143b00, regionState=OPENING, regionLocation=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:36,005 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=c81e1c1af980b616d932866cdd4ff915, regionState=OPENING, regionLocation=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:36,005 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=42e678de61063128349f238929e3d296, regionState=OPENING, regionLocation=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:36,005 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=ac83f7461cc15e741d0440c6ee2781db, regionState=OPENING, regionLocation=3456ee6a3164,33007,1731736285465 2024-11-16T05:54:36,006 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=a6c8ea5676b855dac5a939ace6afadb0, regionState=OPENING, regionLocation=3456ee6a3164,33007,1731736285465 2024-11-16T05:54:36,006 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=bd79de8195c0452a334d7da789e8f23b, regionState=OPENING, regionLocation=3456ee6a3164,33007,1731736285465 2024-11-16T05:54:36,007 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=883c672d1e7de6e08e15c0c42278e66d, regionState=OPENING, regionLocation=3456ee6a3164,33007,1731736285465 2024-11-16T05:54:36,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=10a3e438f5882d261cd799e4d6b75095, ASSIGN because future has completed 2024-11-16T05:54:36,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10a3e438f5882d261cd799e4d6b75095, server=3456ee6a3164,43415,1731736285705}] 2024-11-16T05:54:36,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ec037e054803eda72bad2d81c6143b00, ASSIGN because future has completed 2024-11-16T05:54:36,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c81e1c1af980b616d932866cdd4ff915, ASSIGN because future has completed 2024-11-16T05:54:36,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=42e678de61063128349f238929e3d296, ASSIGN because future has completed 2024-11-16T05:54:36,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42e678de61063128349f238929e3d296, server=3456ee6a3164,43415,1731736285705}] 2024-11-16T05:54:36,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ac83f7461cc15e741d0440c6ee2781db, ASSIGN because future has completed 2024-11-16T05:54:36,013 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a6c8ea5676b855dac5a939ace6afadb0, ASSIGN because future has completed 2024-11-16T05:54:36,014 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bd79de8195c0452a334d7da789e8f23b, ASSIGN because future has completed 2024-11-16T05:54:36,016 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=883c672d1e7de6e08e15c0c42278e66d, ASSIGN because future has completed 2024-11-16T05:54:36,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd79de8195c0452a334d7da789e8f23b, server=3456ee6a3164,33007,1731736285465}] 2024-11-16T05:54:36,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 883c672d1e7de6e08e15c0c42278e66d, server=3456ee6a3164,33007,1731736285465}] 2024-11-16T05:54:36,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a6c8ea5676b855dac5a939ace6afadb0, server=3456ee6a3164,33007,1731736285465}] 2024-11-16T05:54:36,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ac83f7461cc15e741d0440c6ee2781db, server=3456ee6a3164,33007,1731736285465}] 2024-11-16T05:54:36,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c81e1c1af980b616d932866cdd4ff915, server=3456ee6a3164,43415,1731736285705}] 2024-11-16T05:54:36,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure ec037e054803eda72bad2d81c6143b00, server=3456ee6a3164,43415,1731736285705}] 2024-11-16T05:54:36,169 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00. 2024-11-16T05:54:36,169 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => ec037e054803eda72bad2d81c6143b00, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-16T05:54:36,170 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true ec037e054803eda72bad2d81c6143b00 2024-11-16T05:54:36,170 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:36,170 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for ec037e054803eda72bad2d81c6143b00 2024-11-16T05:54:36,170 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for ec037e054803eda72bad2d81c6143b00 2024-11-16T05:54:36,172 INFO [StoreOpener-ec037e054803eda72bad2d81c6143b00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region ec037e054803eda72bad2d81c6143b00 2024-11-16T05:54:36,173 INFO [StoreOpener-ec037e054803eda72bad2d81c6143b00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec037e054803eda72bad2d81c6143b00 columnFamilyName colfamily11 2024-11-16T05:54:36,173 DEBUG [StoreOpener-ec037e054803eda72bad2d81c6143b00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:36,178 INFO [StoreOpener-ec037e054803eda72bad2d81c6143b00-1 {}] regionserver.HStore(327): Store=ec037e054803eda72bad2d81c6143b00/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:36,178 INFO [StoreOpener-ec037e054803eda72bad2d81c6143b00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region ec037e054803eda72bad2d81c6143b00 2024-11-16T05:54:36,179 INFO [StoreOpener-ec037e054803eda72bad2d81c6143b00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec037e054803eda72bad2d81c6143b00 columnFamilyName colfamily21 2024-11-16T05:54:36,180 DEBUG [StoreOpener-ec037e054803eda72bad2d81c6143b00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:36,180 INFO [StoreOpener-ec037e054803eda72bad2d81c6143b00-1 {}] regionserver.HStore(327): Store=ec037e054803eda72bad2d81c6143b00/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:36,180 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b. 2024-11-16T05:54:36,180 INFO [StoreOpener-ec037e054803eda72bad2d81c6143b00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region ec037e054803eda72bad2d81c6143b00 2024-11-16T05:54:36,181 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => bd79de8195c0452a334d7da789e8f23b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-16T05:54:36,181 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true bd79de8195c0452a334d7da789e8f23b 2024-11-16T05:54:36,181 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:36,181 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for bd79de8195c0452a334d7da789e8f23b 2024-11-16T05:54:36,181 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for bd79de8195c0452a334d7da789e8f23b 2024-11-16T05:54:36,182 INFO [StoreOpener-ec037e054803eda72bad2d81c6143b00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec037e054803eda72bad2d81c6143b00 columnFamilyName colfamily31 2024-11-16T05:54:36,182 DEBUG [StoreOpener-ec037e054803eda72bad2d81c6143b00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:36,183 INFO [StoreOpener-bd79de8195c0452a334d7da789e8f23b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region bd79de8195c0452a334d7da789e8f23b 2024-11-16T05:54:36,183 INFO [StoreOpener-ec037e054803eda72bad2d81c6143b00-1 {}] regionserver.HStore(327): Store=ec037e054803eda72bad2d81c6143b00/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:36,183 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for ec037e054803eda72bad2d81c6143b00 2024-11-16T05:54:36,184 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ec037e054803eda72bad2d81c6143b00 2024-11-16T05:54:36,185 INFO [StoreOpener-bd79de8195c0452a334d7da789e8f23b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd79de8195c0452a334d7da789e8f23b columnFamilyName colfamily11 2024-11-16T05:54:36,185 DEBUG [StoreOpener-bd79de8195c0452a334d7da789e8f23b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:36,186 INFO [StoreOpener-bd79de8195c0452a334d7da789e8f23b-1 {}] regionserver.HStore(327): Store=bd79de8195c0452a334d7da789e8f23b/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:36,186 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ec037e054803eda72bad2d81c6143b00 2024-11-16T05:54:36,186 INFO [StoreOpener-bd79de8195c0452a334d7da789e8f23b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region bd79de8195c0452a334d7da789e8f23b 2024-11-16T05:54:36,187 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ec037e054803eda72bad2d81c6143b00/recovered.edits/0000000000000000004 2024-11-16T05:54:36,187 INFO [StoreOpener-bd79de8195c0452a334d7da789e8f23b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd79de8195c0452a334d7da789e8f23b columnFamilyName colfamily21 2024-11-16T05:54:36,187 DEBUG [StoreOpener-bd79de8195c0452a334d7da789e8f23b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:36,188 INFO [StoreOpener-bd79de8195c0452a334d7da789e8f23b-1 {}] regionserver.HStore(327): Store=bd79de8195c0452a334d7da789e8f23b/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:36,188 INFO [StoreOpener-bd79de8195c0452a334d7da789e8f23b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region bd79de8195c0452a334d7da789e8f23b 2024-11-16T05:54:36,190 INFO [StoreOpener-bd79de8195c0452a334d7da789e8f23b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd79de8195c0452a334d7da789e8f23b columnFamilyName colfamily31 2024-11-16T05:54:36,190 DEBUG [StoreOpener-bd79de8195c0452a334d7da789e8f23b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:36,190 INFO [StoreOpener-bd79de8195c0452a334d7da789e8f23b-1 {}] regionserver.HStore(327): Store=bd79de8195c0452a334d7da789e8f23b/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:36,191 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for bd79de8195c0452a334d7da789e8f23b 2024-11-16T05:54:36,192 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bd79de8195c0452a334d7da789e8f23b 2024-11-16T05:54:36,194 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ec037e054803eda72bad2d81c6143b00/recovered.edits/0000000000000000004 2024-11-16T05:54:36,194 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(2902): Flushing ec037e054803eda72bad2d81c6143b00 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T05:54:36,195 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bd79de8195c0452a334d7da789e8f23b 2024-11-16T05:54:36,195 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bd79de8195c0452a334d7da789e8f23b/recovered.edits/0000000000000000004 2024-11-16T05:54:36,203 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bd79de8195c0452a334d7da789e8f23b/recovered.edits/0000000000000000004 2024-11-16T05:54:36,203 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(2902): Flushing bd79de8195c0452a334d7da789e8f23b 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T05:54:36,226 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ec037e054803eda72bad2d81c6143b00/.tmp/colfamily11/397bee99e47f42dea4540cd99fe8e707 is 52, key is ppp/colfamily11:colfamily11/1731736292029/Put/seqid=0 2024-11-16T05:54:36,233 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bd79de8195c0452a334d7da789e8f23b/.tmp/colfamily11/f04e0f5b315f44849d2497a69711d050 is 52, key is mmm/colfamily11:colfamily11/1731736291990/Put/seqid=0 2024-11-16T05:54:36,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741877_1053 (size=39263) 2024-11-16T05:54:36,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741877_1053 (size=39263) 2024-11-16T05:54:36,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741878_1054 (size=39263) 2024-11-16T05:54:36,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741878_1054 (size=39263) 2024-11-16T05:54:36,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741877_1053 (size=39263) 2024-11-16T05:54:36,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741878_1054 (size=39263) 2024-11-16T05:54:36,255 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ec037e054803eda72bad2d81c6143b00/.tmp/colfamily11/397bee99e47f42dea4540cd99fe8e707 2024-11-16T05:54:36,255 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bd79de8195c0452a334d7da789e8f23b/.tmp/colfamily11/f04e0f5b315f44849d2497a69711d050 2024-11-16T05:54:36,264 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ec037e054803eda72bad2d81c6143b00/.tmp/colfamily11/397bee99e47f42dea4540cd99fe8e707 as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ec037e054803eda72bad2d81c6143b00/colfamily11/397bee99e47f42dea4540cd99fe8e707 2024-11-16T05:54:36,265 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bd79de8195c0452a334d7da789e8f23b/.tmp/colfamily11/f04e0f5b315f44849d2497a69711d050 as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bd79de8195c0452a334d7da789e8f23b/colfamily11/f04e0f5b315f44849d2497a69711d050 2024-11-16T05:54:36,275 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ec037e054803eda72bad2d81c6143b00/colfamily11/397bee99e47f42dea4540cd99fe8e707, entries=703, sequenceid=4, filesize=38.3 K 2024-11-16T05:54:36,276 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for ec037e054803eda72bad2d81c6143b00 in 82ms, sequenceid=4, compaction requested=false; wal=null 2024-11-16T05:54:36,276 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bd79de8195c0452a334d7da789e8f23b/colfamily11/f04e0f5b315f44849d2497a69711d050, entries=703, sequenceid=4, filesize=38.3 K 2024-11-16T05:54:36,276 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for bd79de8195c0452a334d7da789e8f23b in 73ms, sequenceid=4, compaction requested=false; wal=null 2024-11-16T05:54:36,277 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ec037e054803eda72bad2d81c6143b00/recovered.edits/0000000000000000004 2024-11-16T05:54:36,278 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bd79de8195c0452a334d7da789e8f23b/recovered.edits/0000000000000000004 2024-11-16T05:54:36,279 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for ec037e054803eda72bad2d81c6143b00 2024-11-16T05:54:36,279 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for ec037e054803eda72bad2d81c6143b00 2024-11-16T05:54:36,279 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for bd79de8195c0452a334d7da789e8f23b 2024-11-16T05:54:36,279 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for bd79de8195c0452a334d7da789e8f23b 2024-11-16T05:54:36,280 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:54:36,280 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:54:36,282 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for ec037e054803eda72bad2d81c6143b00 2024-11-16T05:54:36,283 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for bd79de8195c0452a334d7da789e8f23b 2024-11-16T05:54:36,286 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bd79de8195c0452a334d7da789e8f23b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:54:36,287 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened bd79de8195c0452a334d7da789e8f23b; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60727051, jitterRate=-0.09509642422199249}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:54:36,288 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bd79de8195c0452a334d7da789e8f23b 2024-11-16T05:54:36,288 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for bd79de8195c0452a334d7da789e8f23b: Running coprocessor pre-open hook at 1731736476181Writing region info on filesystem at 1731736476181Initializing all the Stores at 1731736476182 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736476182Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736476183 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736476183Obtaining lock to block concurrent updates at 1731736476203 (+20 ms)Preparing flush snapshotting stores in bd79de8195c0452a334d7da789e8f23b at 1731736476203Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731736476203Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b. at 1731736476203Flushing bd79de8195c0452a334d7da789e8f23b/colfamily11: creating writer at 1731736476203Flushing bd79de8195c0452a334d7da789e8f23b/colfamily11: appending metadata at 1731736476233 (+30 ms)Flushing bd79de8195c0452a334d7da789e8f23b/colfamily11: closing flushed file at 1731736476233Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b5c65ae: reopening flushed file at 1731736476264 (+31 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for bd79de8195c0452a334d7da789e8f23b in 73ms, sequenceid=4, compaction requested=false; wal=null at 1731736476276 (+12 ms)Cleaning up temporary data from old regions at 1731736476279 (+3 ms)Running coprocessor post-open hooks at 1731736476288 (+9 ms)Region opened successfully at 1731736476288 2024-11-16T05:54:36,289 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b., pid=74, masterSystemTime=1731736476167 2024-11-16T05:54:36,290 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b. 2024-11-16T05:54:36,291 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b. 2024-11-16T05:54:36,291 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d. 2024-11-16T05:54:36,291 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 883c672d1e7de6e08e15c0c42278e66d, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-16T05:54:36,291 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=bd79de8195c0452a334d7da789e8f23b, regionState=OPEN, openSeqNum=5, regionLocation=3456ee6a3164,33007,1731736285465 2024-11-16T05:54:36,291 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 883c672d1e7de6e08e15c0c42278e66d 2024-11-16T05:54:36,292 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:36,292 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 883c672d1e7de6e08e15c0c42278e66d 2024-11-16T05:54:36,292 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 883c672d1e7de6e08e15c0c42278e66d 2024-11-16T05:54:36,295 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ec037e054803eda72bad2d81c6143b00/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:54:36,296 INFO [StoreOpener-883c672d1e7de6e08e15c0c42278e66d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 883c672d1e7de6e08e15c0c42278e66d 2024-11-16T05:54:36,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd79de8195c0452a334d7da789e8f23b, server=3456ee6a3164,33007,1731736285465 because future has completed 2024-11-16T05:54:36,297 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened ec037e054803eda72bad2d81c6143b00; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74837500, jitterRate=0.11516565084457397}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:54:36,297 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ec037e054803eda72bad2d81c6143b00 2024-11-16T05:54:36,297 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for ec037e054803eda72bad2d81c6143b00: Running coprocessor pre-open hook at 1731736476170Writing region info on filesystem at 1731736476170Initializing all the Stores at 1731736476171 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736476171Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736476172 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736476172Obtaining lock to block concurrent updates at 1731736476194 (+22 ms)Preparing flush snapshotting stores in ec037e054803eda72bad2d81c6143b00 at 1731736476194Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731736476194Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00. at 1731736476194Flushing ec037e054803eda72bad2d81c6143b00/colfamily11: creating writer at 1731736476195 (+1 ms)Flushing ec037e054803eda72bad2d81c6143b00/colfamily11: appending metadata at 1731736476225 (+30 ms)Flushing ec037e054803eda72bad2d81c6143b00/colfamily11: closing flushed file at 1731736476225Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ded2a60: reopening flushed file at 1731736476263 (+38 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for ec037e054803eda72bad2d81c6143b00 in 82ms, sequenceid=4, compaction requested=false; wal=null at 1731736476276 (+13 ms)Cleaning up temporary data from old regions at 1731736476279 (+3 ms)Running coprocessor post-open hooks at 1731736476297 (+18 ms)Region opened successfully at 1731736476297 2024-11-16T05:54:36,297 INFO [StoreOpener-883c672d1e7de6e08e15c0c42278e66d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 883c672d1e7de6e08e15c0c42278e66d columnFamilyName colfamily11 2024-11-16T05:54:36,297 DEBUG [StoreOpener-883c672d1e7de6e08e15c0c42278e66d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:36,298 INFO [StoreOpener-883c672d1e7de6e08e15c0c42278e66d-1 {}] regionserver.HStore(327): Store=883c672d1e7de6e08e15c0c42278e66d/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:36,299 INFO [StoreOpener-883c672d1e7de6e08e15c0c42278e66d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 883c672d1e7de6e08e15c0c42278e66d 2024-11-16T05:54:36,300 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00., pid=79, masterSystemTime=1731736476163 2024-11-16T05:54:36,300 INFO [StoreOpener-883c672d1e7de6e08e15c0c42278e66d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 883c672d1e7de6e08e15c0c42278e66d columnFamilyName colfamily21 2024-11-16T05:54:36,300 DEBUG [StoreOpener-883c672d1e7de6e08e15c0c42278e66d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:36,303 INFO [StoreOpener-883c672d1e7de6e08e15c0c42278e66d-1 {}] regionserver.HStore(327): Store=883c672d1e7de6e08e15c0c42278e66d/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:36,303 INFO [StoreOpener-883c672d1e7de6e08e15c0c42278e66d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 883c672d1e7de6e08e15c0c42278e66d 2024-11-16T05:54:36,305 INFO [StoreOpener-883c672d1e7de6e08e15c0c42278e66d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 883c672d1e7de6e08e15c0c42278e66d columnFamilyName colfamily31 2024-11-16T05:54:36,305 DEBUG [StoreOpener-883c672d1e7de6e08e15c0c42278e66d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:36,306 INFO [StoreOpener-883c672d1e7de6e08e15c0c42278e66d-1 {}] regionserver.HStore(327): Store=883c672d1e7de6e08e15c0c42278e66d/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:36,306 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 883c672d1e7de6e08e15c0c42278e66d 2024-11-16T05:54:36,307 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/883c672d1e7de6e08e15c0c42278e66d 2024-11-16T05:54:36,309 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00. 2024-11-16T05:54:36,310 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00. 2024-11-16T05:54:36,310 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095. 2024-11-16T05:54:36,310 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7752): Opening region: {ENCODED => 10a3e438f5882d261cd799e4d6b75095, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-16T05:54:36,310 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 10a3e438f5882d261cd799e4d6b75095 2024-11-16T05:54:36,311 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:36,311 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7794): checking encryption for 10a3e438f5882d261cd799e4d6b75095 2024-11-16T05:54:36,311 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7797): checking classloading for 10a3e438f5882d261cd799e4d6b75095 2024-11-16T05:54:36,312 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/883c672d1e7de6e08e15c0c42278e66d 2024-11-16T05:54:36,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=69 2024-11-16T05:54:36,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure bd79de8195c0452a334d7da789e8f23b, server=3456ee6a3164,33007,1731736285465 in 283 msec 2024-11-16T05:54:36,313 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/883c672d1e7de6e08e15c0c42278e66d/recovered.edits/0000000000000000004 2024-11-16T05:54:36,317 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bd79de8195c0452a334d7da789e8f23b, ASSIGN in 463 msec 2024-11-16T05:54:36,319 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=ec037e054803eda72bad2d81c6143b00, regionState=OPEN, openSeqNum=5, regionLocation=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:36,320 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/883c672d1e7de6e08e15c0c42278e66d/recovered.edits/0000000000000000004 2024-11-16T05:54:36,321 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(2902): Flushing 883c672d1e7de6e08e15c0c42278e66d 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T05:54:36,322 INFO [StoreOpener-10a3e438f5882d261cd799e4d6b75095-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 10a3e438f5882d261cd799e4d6b75095 2024-11-16T05:54:36,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure ec037e054803eda72bad2d81c6143b00, server=3456ee6a3164,43415,1731736285705 because future has completed 2024-11-16T05:54:36,325 INFO [StoreOpener-10a3e438f5882d261cd799e4d6b75095-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10a3e438f5882d261cd799e4d6b75095 columnFamilyName colfamily11 2024-11-16T05:54:36,325 DEBUG [StoreOpener-10a3e438f5882d261cd799e4d6b75095-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:36,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=67 2024-11-16T05:54:36,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure ec037e054803eda72bad2d81c6143b00, server=3456ee6a3164,43415,1731736285705 in 297 msec 2024-11-16T05:54:36,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ec037e054803eda72bad2d81c6143b00, ASSIGN in 479 msec 2024-11-16T05:54:36,337 INFO [StoreOpener-10a3e438f5882d261cd799e4d6b75095-1 {}] regionserver.HStore(327): Store=10a3e438f5882d261cd799e4d6b75095/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:36,337 INFO [StoreOpener-10a3e438f5882d261cd799e4d6b75095-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 10a3e438f5882d261cd799e4d6b75095 2024-11-16T05:54:36,338 INFO [StoreOpener-10a3e438f5882d261cd799e4d6b75095-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10a3e438f5882d261cd799e4d6b75095 columnFamilyName colfamily21 2024-11-16T05:54:36,339 DEBUG [StoreOpener-10a3e438f5882d261cd799e4d6b75095-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:36,340 INFO [StoreOpener-10a3e438f5882d261cd799e4d6b75095-1 {}] regionserver.HStore(327): Store=10a3e438f5882d261cd799e4d6b75095/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:36,340 INFO [StoreOpener-10a3e438f5882d261cd799e4d6b75095-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 10a3e438f5882d261cd799e4d6b75095 2024-11-16T05:54:36,341 INFO [StoreOpener-10a3e438f5882d261cd799e4d6b75095-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 10a3e438f5882d261cd799e4d6b75095 columnFamilyName colfamily31 2024-11-16T05:54:36,341 DEBUG [StoreOpener-10a3e438f5882d261cd799e4d6b75095-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:36,342 INFO [StoreOpener-10a3e438f5882d261cd799e4d6b75095-1 {}] regionserver.HStore(327): Store=10a3e438f5882d261cd799e4d6b75095/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:36,342 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1038): replaying wal for 10a3e438f5882d261cd799e4d6b75095 2024-11-16T05:54:36,343 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10a3e438f5882d261cd799e4d6b75095 2024-11-16T05:54:36,345 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10a3e438f5882d261cd799e4d6b75095 2024-11-16T05:54:36,346 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10a3e438f5882d261cd799e4d6b75095/recovered.edits/0000000000000000004 2024-11-16T05:54:36,346 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/883c672d1e7de6e08e15c0c42278e66d/.tmp/colfamily11/e55293f97b4243739301b80aeeb423da is 52, key is ggg/colfamily11:colfamily11/1731736291905/Put/seqid=0 2024-11-16T05:54:36,352 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10a3e438f5882d261cd799e4d6b75095/recovered.edits/0000000000000000004 2024-11-16T05:54:36,352 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(2902): Flushing 10a3e438f5882d261cd799e4d6b75095 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T05:54:36,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741879_1055 (size=39263) 2024-11-16T05:54:36,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741879_1055 (size=39263) 2024-11-16T05:54:36,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741879_1055 (size=39263) 2024-11-16T05:54:36,385 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/883c672d1e7de6e08e15c0c42278e66d/.tmp/colfamily11/e55293f97b4243739301b80aeeb423da 2024-11-16T05:54:36,387 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10a3e438f5882d261cd799e4d6b75095/.tmp/colfamily11/3069a10d62c740149abf9cc05983064b is 52, key is vvv/colfamily11:colfamily11/1731736292105/Put/seqid=0 2024-11-16T05:54:36,408 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/883c672d1e7de6e08e15c0c42278e66d/.tmp/colfamily11/e55293f97b4243739301b80aeeb423da as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/883c672d1e7de6e08e15c0c42278e66d/colfamily11/e55293f97b4243739301b80aeeb423da 2024-11-16T05:54:36,418 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/883c672d1e7de6e08e15c0c42278e66d/colfamily11/e55293f97b4243739301b80aeeb423da, entries=703, sequenceid=4, filesize=38.3 K 2024-11-16T05:54:36,418 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 883c672d1e7de6e08e15c0c42278e66d in 98ms, sequenceid=4, compaction requested=false; wal=null 2024-11-16T05:54:36,419 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/883c672d1e7de6e08e15c0c42278e66d/recovered.edits/0000000000000000004 2024-11-16T05:54:36,420 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 883c672d1e7de6e08e15c0c42278e66d 2024-11-16T05:54:36,420 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 883c672d1e7de6e08e15c0c42278e66d 2024-11-16T05:54:36,421 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:54:36,423 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 883c672d1e7de6e08e15c0c42278e66d 2024-11-16T05:54:36,430 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/883c672d1e7de6e08e15c0c42278e66d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:54:36,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741880_1056 (size=39263) 2024-11-16T05:54:36,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741880_1056 (size=39263) 2024-11-16T05:54:36,432 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 883c672d1e7de6e08e15c0c42278e66d; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66795490, jitterRate=-0.004669636487960815}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:54:36,432 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 883c672d1e7de6e08e15c0c42278e66d 2024-11-16T05:54:36,432 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 883c672d1e7de6e08e15c0c42278e66d: Running coprocessor pre-open hook at 1731736476292Writing region info on filesystem at 1731736476292Initializing all the Stores at 1731736476295 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736476296 (+1 ms)Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736476296Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736476296Obtaining lock to block concurrent updates at 1731736476321 (+25 ms)Preparing flush snapshotting stores in 883c672d1e7de6e08e15c0c42278e66d at 1731736476321Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731736476321Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d. at 1731736476321Flushing 883c672d1e7de6e08e15c0c42278e66d/colfamily11: creating writer at 1731736476321Flushing 883c672d1e7de6e08e15c0c42278e66d/colfamily11: appending metadata at 1731736476346 (+25 ms)Flushing 883c672d1e7de6e08e15c0c42278e66d/colfamily11: closing flushed file at 1731736476346Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f0557cf: reopening flushed file at 1731736476400 (+54 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 883c672d1e7de6e08e15c0c42278e66d in 98ms, sequenceid=4, compaction requested=false; wal=null at 1731736476418 (+18 ms)Cleaning up temporary data from old regions at 1731736476420 (+2 ms)Running coprocessor post-open hooks at 1731736476432 (+12 ms)Region opened successfully at 1731736476432 2024-11-16T05:54:36,433 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d., pid=75, masterSystemTime=1731736476167 2024-11-16T05:54:36,435 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10a3e438f5882d261cd799e4d6b75095/.tmp/colfamily11/3069a10d62c740149abf9cc05983064b 2024-11-16T05:54:36,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741880_1056 (size=39263) 2024-11-16T05:54:36,436 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d. 2024-11-16T05:54:36,436 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d. 2024-11-16T05:54:36,436 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db. 2024-11-16T05:54:36,436 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => ac83f7461cc15e741d0440c6ee2781db, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-16T05:54:36,436 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true ac83f7461cc15e741d0440c6ee2781db 2024-11-16T05:54:36,437 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=883c672d1e7de6e08e15c0c42278e66d, regionState=OPEN, openSeqNum=5, regionLocation=3456ee6a3164,33007,1731736285465 2024-11-16T05:54:36,437 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:36,437 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for ac83f7461cc15e741d0440c6ee2781db 2024-11-16T05:54:36,437 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for ac83f7461cc15e741d0440c6ee2781db 2024-11-16T05:54:36,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=75, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 883c672d1e7de6e08e15c0c42278e66d, server=3456ee6a3164,33007,1731736285465 because future has completed 2024-11-16T05:54:36,440 INFO [StoreOpener-ac83f7461cc15e741d0440c6ee2781db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region ac83f7461cc15e741d0440c6ee2781db 2024-11-16T05:54:36,441 INFO [StoreOpener-ac83f7461cc15e741d0440c6ee2781db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac83f7461cc15e741d0440c6ee2781db columnFamilyName colfamily11 2024-11-16T05:54:36,441 DEBUG [StoreOpener-ac83f7461cc15e741d0440c6ee2781db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:36,442 INFO [StoreOpener-ac83f7461cc15e741d0440c6ee2781db-1 {}] regionserver.HStore(327): Store=ac83f7461cc15e741d0440c6ee2781db/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:36,442 INFO [StoreOpener-ac83f7461cc15e741d0440c6ee2781db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region ac83f7461cc15e741d0440c6ee2781db 2024-11-16T05:54:36,444 INFO [StoreOpener-ac83f7461cc15e741d0440c6ee2781db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac83f7461cc15e741d0440c6ee2781db columnFamilyName colfamily21 2024-11-16T05:54:36,444 DEBUG [StoreOpener-ac83f7461cc15e741d0440c6ee2781db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:36,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=65 2024-11-16T05:54:36,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=65, state=SUCCESS, hasLock=false; OpenRegionProcedure 883c672d1e7de6e08e15c0c42278e66d, server=3456ee6a3164,33007,1731736285465 in 423 msec 2024-11-16T05:54:36,446 INFO [StoreOpener-ac83f7461cc15e741d0440c6ee2781db-1 {}] regionserver.HStore(327): Store=ac83f7461cc15e741d0440c6ee2781db/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:36,446 INFO [StoreOpener-ac83f7461cc15e741d0440c6ee2781db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region ac83f7461cc15e741d0440c6ee2781db 2024-11-16T05:54:36,446 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10a3e438f5882d261cd799e4d6b75095/.tmp/colfamily11/3069a10d62c740149abf9cc05983064b as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10a3e438f5882d261cd799e4d6b75095/colfamily11/3069a10d62c740149abf9cc05983064b 2024-11-16T05:54:36,447 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=883c672d1e7de6e08e15c0c42278e66d, ASSIGN in 595 msec 2024-11-16T05:54:36,448 INFO [StoreOpener-ac83f7461cc15e741d0440c6ee2781db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac83f7461cc15e741d0440c6ee2781db columnFamilyName colfamily31 2024-11-16T05:54:36,448 DEBUG [StoreOpener-ac83f7461cc15e741d0440c6ee2781db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:36,448 INFO [StoreOpener-ac83f7461cc15e741d0440c6ee2781db-1 {}] regionserver.HStore(327): Store=ac83f7461cc15e741d0440c6ee2781db/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:36,449 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for ac83f7461cc15e741d0440c6ee2781db 2024-11-16T05:54:36,449 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac83f7461cc15e741d0440c6ee2781db 2024-11-16T05:54:36,451 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac83f7461cc15e741d0440c6ee2781db 2024-11-16T05:54:36,452 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac83f7461cc15e741d0440c6ee2781db/recovered.edits/0000000000000000004 2024-11-16T05:54:36,454 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10a3e438f5882d261cd799e4d6b75095/colfamily11/3069a10d62c740149abf9cc05983064b, entries=703, sequenceid=4, filesize=38.3 K 2024-11-16T05:54:36,454 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 10a3e438f5882d261cd799e4d6b75095 in 102ms, sequenceid=4, compaction requested=false; wal=null 2024-11-16T05:54:36,455 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10a3e438f5882d261cd799e4d6b75095/recovered.edits/0000000000000000004 2024-11-16T05:54:36,456 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1048): stopping wal replay for 10a3e438f5882d261cd799e4d6b75095 2024-11-16T05:54:36,457 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1060): Cleaning up temporary data for 10a3e438f5882d261cd799e4d6b75095 2024-11-16T05:54:36,457 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac83f7461cc15e741d0440c6ee2781db/recovered.edits/0000000000000000004 2024-11-16T05:54:36,457 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(2902): Flushing ac83f7461cc15e741d0440c6ee2781db 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T05:54:36,457 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:54:36,459 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1093): writing seq id for 10a3e438f5882d261cd799e4d6b75095 2024-11-16T05:54:36,462 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10a3e438f5882d261cd799e4d6b75095/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:54:36,464 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1114): Opened 10a3e438f5882d261cd799e4d6b75095; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61394579, jitterRate=-0.08514948189258575}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:54:36,464 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 10a3e438f5882d261cd799e4d6b75095 2024-11-16T05:54:36,464 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1006): Region open journal for 10a3e438f5882d261cd799e4d6b75095: Running coprocessor pre-open hook at 1731736476311Writing region info on filesystem at 1731736476311Initializing all the Stores at 1731736476315 (+4 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736476315Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736476322 (+7 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736476322Obtaining lock to block concurrent updates at 1731736476352 (+30 ms)Preparing flush snapshotting stores in 10a3e438f5882d261cd799e4d6b75095 at 1731736476352Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731736476352Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095. at 1731736476352Flushing 10a3e438f5882d261cd799e4d6b75095/colfamily11: creating writer at 1731736476352Flushing 10a3e438f5882d261cd799e4d6b75095/colfamily11: appending metadata at 1731736476386 (+34 ms)Flushing 10a3e438f5882d261cd799e4d6b75095/colfamily11: closing flushed file at 1731736476386Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d5289e5: reopening flushed file at 1731736476445 (+59 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 10a3e438f5882d261cd799e4d6b75095 in 102ms, sequenceid=4, compaction requested=false; wal=null at 1731736476454 (+9 ms)Cleaning up temporary data from old regions at 1731736476457 (+3 ms)Running coprocessor post-open hooks at 1731736476464 (+7 ms)Region opened successfully at 1731736476464 2024-11-16T05:54:36,465 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095., pid=72, masterSystemTime=1731736476163 2024-11-16T05:54:36,470 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095. 2024-11-16T05:54:36,470 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095. 2024-11-16T05:54:36,470 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296. 2024-11-16T05:54:36,470 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => 42e678de61063128349f238929e3d296, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-16T05:54:36,471 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 42e678de61063128349f238929e3d296 2024-11-16T05:54:36,471 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:36,471 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for 42e678de61063128349f238929e3d296 2024-11-16T05:54:36,471 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for 42e678de61063128349f238929e3d296 2024-11-16T05:54:36,472 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=10a3e438f5882d261cd799e4d6b75095, regionState=OPEN, openSeqNum=5, regionLocation=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:36,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 10a3e438f5882d261cd799e4d6b75095, server=3456ee6a3164,43415,1731736285705 because future has completed 2024-11-16T05:54:36,476 INFO [StoreOpener-42e678de61063128349f238929e3d296-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 42e678de61063128349f238929e3d296 2024-11-16T05:54:36,477 INFO [StoreOpener-42e678de61063128349f238929e3d296-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42e678de61063128349f238929e3d296 columnFamilyName colfamily11 2024-11-16T05:54:36,477 DEBUG [StoreOpener-42e678de61063128349f238929e3d296-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:36,478 INFO [StoreOpener-42e678de61063128349f238929e3d296-1 {}] regionserver.HStore(327): Store=42e678de61063128349f238929e3d296/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:36,478 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac83f7461cc15e741d0440c6ee2781db/.tmp/colfamily11/98dbb413ddb745eeb8e80021c7dab1d0 is 52, key is xxx/colfamily11:colfamily11/1731736292143/Put/seqid=0 2024-11-16T05:54:36,478 INFO [StoreOpener-42e678de61063128349f238929e3d296-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 42e678de61063128349f238929e3d296 2024-11-16T05:54:36,480 INFO [StoreOpener-42e678de61063128349f238929e3d296-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42e678de61063128349f238929e3d296 columnFamilyName colfamily21 2024-11-16T05:54:36,480 DEBUG [StoreOpener-42e678de61063128349f238929e3d296-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:36,481 INFO [StoreOpener-42e678de61063128349f238929e3d296-1 {}] regionserver.HStore(327): Store=42e678de61063128349f238929e3d296/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:36,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=72, resume processing ppid=71 2024-11-16T05:54:36,481 INFO [StoreOpener-42e678de61063128349f238929e3d296-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 42e678de61063128349f238929e3d296 2024-11-16T05:54:36,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, ppid=71, state=SUCCESS, hasLock=false; OpenRegionProcedure 10a3e438f5882d261cd799e4d6b75095, server=3456ee6a3164,43415,1731736285705 in 469 msec 2024-11-16T05:54:36,483 INFO [StoreOpener-42e678de61063128349f238929e3d296-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 42e678de61063128349f238929e3d296 columnFamilyName colfamily31 2024-11-16T05:54:36,483 DEBUG [StoreOpener-42e678de61063128349f238929e3d296-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:36,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=10a3e438f5882d261cd799e4d6b75095, ASSIGN in 631 msec 2024-11-16T05:54:36,485 INFO [StoreOpener-42e678de61063128349f238929e3d296-1 {}] regionserver.HStore(327): Store=42e678de61063128349f238929e3d296/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:36,485 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for 42e678de61063128349f238929e3d296 2024-11-16T05:54:36,486 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/42e678de61063128349f238929e3d296 2024-11-16T05:54:36,488 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/42e678de61063128349f238929e3d296 2024-11-16T05:54:36,488 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/42e678de61063128349f238929e3d296/recovered.edits/0000000000000000004 2024-11-16T05:54:36,495 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/42e678de61063128349f238929e3d296/recovered.edits/0000000000000000004 2024-11-16T05:54:36,495 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(2902): Flushing 42e678de61063128349f238929e3d296 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T05:54:36,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741881_1057 (size=39263) 2024-11-16T05:54:36,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741881_1057 (size=39263) 2024-11-16T05:54:36,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741881_1057 (size=39263) 2024-11-16T05:54:36,504 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac83f7461cc15e741d0440c6ee2781db/.tmp/colfamily11/98dbb413ddb745eeb8e80021c7dab1d0 2024-11-16T05:54:36,513 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac83f7461cc15e741d0440c6ee2781db/.tmp/colfamily11/98dbb413ddb745eeb8e80021c7dab1d0 as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac83f7461cc15e741d0440c6ee2781db/colfamily11/98dbb413ddb745eeb8e80021c7dab1d0 2024-11-16T05:54:36,518 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/42e678de61063128349f238929e3d296/.tmp/colfamily11/591373ee861a4b83a7506c250451c60d is 52, key is ddd/colfamily11:colfamily11/1731736291788/Put/seqid=0 2024-11-16T05:54:36,520 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac83f7461cc15e741d0440c6ee2781db/colfamily11/98dbb413ddb745eeb8e80021c7dab1d0, entries=703, sequenceid=4, filesize=38.3 K 2024-11-16T05:54:36,521 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for ac83f7461cc15e741d0440c6ee2781db in 63ms, sequenceid=4, compaction requested=false; wal=null 2024-11-16T05:54:36,521 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac83f7461cc15e741d0440c6ee2781db/recovered.edits/0000000000000000004 2024-11-16T05:54:36,523 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for ac83f7461cc15e741d0440c6ee2781db 2024-11-16T05:54:36,523 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for ac83f7461cc15e741d0440c6ee2781db 2024-11-16T05:54:36,523 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:54:36,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741882_1058 (size=39263) 2024-11-16T05:54:36,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741882_1058 (size=39263) 2024-11-16T05:54:36,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741882_1058 (size=39263) 2024-11-16T05:54:36,526 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/42e678de61063128349f238929e3d296/.tmp/colfamily11/591373ee861a4b83a7506c250451c60d 2024-11-16T05:54:36,527 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for ac83f7461cc15e741d0440c6ee2781db 2024-11-16T05:54:36,529 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac83f7461cc15e741d0440c6ee2781db/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:54:36,530 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened ac83f7461cc15e741d0440c6ee2781db; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67910771, jitterRate=0.011949345469474792}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:54:36,530 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ac83f7461cc15e741d0440c6ee2781db 2024-11-16T05:54:36,530 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for ac83f7461cc15e741d0440c6ee2781db: Running coprocessor pre-open hook at 1731736476437Writing region info on filesystem at 1731736476437Initializing all the Stores at 1731736476439 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736476439Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736476439Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736476439Obtaining lock to block concurrent updates at 1731736476457 (+18 ms)Preparing flush snapshotting stores in ac83f7461cc15e741d0440c6ee2781db at 1731736476457Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731736476457Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db. at 1731736476457Flushing ac83f7461cc15e741d0440c6ee2781db/colfamily11: creating writer at 1731736476457Flushing ac83f7461cc15e741d0440c6ee2781db/colfamily11: appending metadata at 1731736476478 (+21 ms)Flushing ac83f7461cc15e741d0440c6ee2781db/colfamily11: closing flushed file at 1731736476478Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e398919: reopening flushed file at 1731736476512 (+34 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for ac83f7461cc15e741d0440c6ee2781db in 63ms, sequenceid=4, compaction requested=false; wal=null at 1731736476521 (+9 ms)Cleaning up temporary data from old regions at 1731736476523 (+2 ms)Running coprocessor post-open hooks at 1731736476530 (+7 ms)Region opened successfully at 1731736476530 2024-11-16T05:54:36,531 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db., pid=77, masterSystemTime=1731736476167 2024-11-16T05:54:36,533 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db. 2024-11-16T05:54:36,533 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db. 2024-11-16T05:54:36,533 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0. 2024-11-16T05:54:36,533 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => a6c8ea5676b855dac5a939ace6afadb0, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-16T05:54:36,534 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=ac83f7461cc15e741d0440c6ee2781db, regionState=OPEN, openSeqNum=5, regionLocation=3456ee6a3164,33007,1731736285465 2024-11-16T05:54:36,534 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true a6c8ea5676b855dac5a939ace6afadb0 2024-11-16T05:54:36,534 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:36,534 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for a6c8ea5676b855dac5a939ace6afadb0 2024-11-16T05:54:36,534 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for a6c8ea5676b855dac5a939ace6afadb0 2024-11-16T05:54:36,534 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/42e678de61063128349f238929e3d296/.tmp/colfamily11/591373ee861a4b83a7506c250451c60d as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/42e678de61063128349f238929e3d296/colfamily11/591373ee861a4b83a7506c250451c60d 2024-11-16T05:54:36,537 INFO [StoreOpener-a6c8ea5676b855dac5a939ace6afadb0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region a6c8ea5676b855dac5a939ace6afadb0 2024-11-16T05:54:36,537 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ac83f7461cc15e741d0440c6ee2781db, server=3456ee6a3164,33007,1731736285465 because future has completed 2024-11-16T05:54:36,539 INFO [StoreOpener-a6c8ea5676b855dac5a939ace6afadb0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6c8ea5676b855dac5a939ace6afadb0 columnFamilyName colfamily11 2024-11-16T05:54:36,539 DEBUG [StoreOpener-a6c8ea5676b855dac5a939ace6afadb0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:36,540 INFO [StoreOpener-a6c8ea5676b855dac5a939ace6afadb0-1 {}] regionserver.HStore(327): Store=a6c8ea5676b855dac5a939ace6afadb0/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:36,540 INFO [StoreOpener-a6c8ea5676b855dac5a939ace6afadb0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region a6c8ea5676b855dac5a939ace6afadb0 2024-11-16T05:54:36,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=66 2024-11-16T05:54:36,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure ac83f7461cc15e741d0440c6ee2781db, server=3456ee6a3164,33007,1731736285465 in 515 msec 2024-11-16T05:54:36,542 INFO [StoreOpener-a6c8ea5676b855dac5a939ace6afadb0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6c8ea5676b855dac5a939ace6afadb0 columnFamilyName colfamily21 2024-11-16T05:54:36,543 DEBUG [StoreOpener-a6c8ea5676b855dac5a939ace6afadb0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:36,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ac83f7461cc15e741d0440c6ee2781db, ASSIGN in 691 msec 2024-11-16T05:54:36,543 INFO [StoreOpener-a6c8ea5676b855dac5a939ace6afadb0-1 {}] regionserver.HStore(327): Store=a6c8ea5676b855dac5a939ace6afadb0/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:36,543 INFO [StoreOpener-a6c8ea5676b855dac5a939ace6afadb0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region a6c8ea5676b855dac5a939ace6afadb0 2024-11-16T05:54:36,544 INFO [StoreOpener-a6c8ea5676b855dac5a939ace6afadb0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a6c8ea5676b855dac5a939ace6afadb0 columnFamilyName colfamily31 2024-11-16T05:54:36,544 DEBUG [StoreOpener-a6c8ea5676b855dac5a939ace6afadb0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:36,545 INFO [StoreOpener-a6c8ea5676b855dac5a939ace6afadb0-1 {}] regionserver.HStore(327): Store=a6c8ea5676b855dac5a939ace6afadb0/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:36,545 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for a6c8ea5676b855dac5a939ace6afadb0 2024-11-16T05:54:36,546 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a6c8ea5676b855dac5a939ace6afadb0 2024-11-16T05:54:36,546 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/42e678de61063128349f238929e3d296/colfamily11/591373ee861a4b83a7506c250451c60d, entries=703, sequenceid=4, filesize=38.3 K 2024-11-16T05:54:36,547 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 42e678de61063128349f238929e3d296 in 52ms, sequenceid=4, compaction requested=false; wal=null 2024-11-16T05:54:36,547 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a6c8ea5676b855dac5a939ace6afadb0 2024-11-16T05:54:36,548 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/42e678de61063128349f238929e3d296/recovered.edits/0000000000000000004 2024-11-16T05:54:36,548 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a6c8ea5676b855dac5a939ace6afadb0/recovered.edits/0000000000000000004 2024-11-16T05:54:36,549 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for 42e678de61063128349f238929e3d296 2024-11-16T05:54:36,549 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for 42e678de61063128349f238929e3d296 2024-11-16T05:54:36,549 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:54:36,551 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for 42e678de61063128349f238929e3d296 2024-11-16T05:54:36,552 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/42e678de61063128349f238929e3d296/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:54:36,553 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened 42e678de61063128349f238929e3d296; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73627504, jitterRate=0.09713530540466309}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:54:36,554 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 42e678de61063128349f238929e3d296 2024-11-16T05:54:36,554 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for 42e678de61063128349f238929e3d296: Running coprocessor pre-open hook at 1731736476471Writing region info on filesystem at 1731736476471Initializing all the Stores at 1731736476474 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736476474Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736476475 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736476475Obtaining lock to block concurrent updates at 1731736476495 (+20 ms)Preparing flush snapshotting stores in 42e678de61063128349f238929e3d296 at 1731736476495Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731736476495Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296. at 1731736476495Flushing 42e678de61063128349f238929e3d296/colfamily11: creating writer at 1731736476495Flushing 42e678de61063128349f238929e3d296/colfamily11: appending metadata at 1731736476517 (+22 ms)Flushing 42e678de61063128349f238929e3d296/colfamily11: closing flushed file at 1731736476517Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@772816e1: reopening flushed file at 1731736476533 (+16 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 42e678de61063128349f238929e3d296 in 52ms, sequenceid=4, compaction requested=false; wal=null at 1731736476547 (+14 ms)Cleaning up temporary data from old regions at 1731736476549 (+2 ms)Running coprocessor post-open hooks at 1731736476554 (+5 ms)Region opened successfully at 1731736476554 2024-11-16T05:54:36,554 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a6c8ea5676b855dac5a939ace6afadb0/recovered.edits/0000000000000000004 2024-11-16T05:54:36,554 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(2902): Flushing a6c8ea5676b855dac5a939ace6afadb0 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T05:54:36,558 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296., pid=73, masterSystemTime=1731736476163 2024-11-16T05:54:36,559 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296. 2024-11-16T05:54:36,559 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296. 2024-11-16T05:54:36,560 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915. 2024-11-16T05:54:36,560 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => c81e1c1af980b616d932866cdd4ff915, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-16T05:54:36,560 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=42e678de61063128349f238929e3d296, regionState=OPEN, openSeqNum=5, regionLocation=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:36,560 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true c81e1c1af980b616d932866cdd4ff915 2024-11-16T05:54:36,560 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T05:54:36,560 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for c81e1c1af980b616d932866cdd4ff915 2024-11-16T05:54:36,560 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for c81e1c1af980b616d932866cdd4ff915 2024-11-16T05:54:36,562 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 42e678de61063128349f238929e3d296, server=3456ee6a3164,43415,1731736285705 because future has completed 2024-11-16T05:54:36,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=68 2024-11-16T05:54:36,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 42e678de61063128349f238929e3d296, server=3456ee6a3164,43415,1731736285705 in 551 msec 2024-11-16T05:54:36,573 INFO [StoreOpener-c81e1c1af980b616d932866cdd4ff915-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region c81e1c1af980b616d932866cdd4ff915 2024-11-16T05:54:36,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=42e678de61063128349f238929e3d296, ASSIGN in 717 msec 2024-11-16T05:54:36,575 INFO [StoreOpener-c81e1c1af980b616d932866cdd4ff915-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c81e1c1af980b616d932866cdd4ff915 columnFamilyName colfamily11 2024-11-16T05:54:36,575 DEBUG [StoreOpener-c81e1c1af980b616d932866cdd4ff915-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:36,576 INFO [StoreOpener-c81e1c1af980b616d932866cdd4ff915-1 {}] regionserver.HStore(327): Store=c81e1c1af980b616d932866cdd4ff915/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:36,576 INFO [StoreOpener-c81e1c1af980b616d932866cdd4ff915-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region c81e1c1af980b616d932866cdd4ff915 2024-11-16T05:54:36,578 INFO [StoreOpener-c81e1c1af980b616d932866cdd4ff915-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c81e1c1af980b616d932866cdd4ff915 columnFamilyName colfamily21 2024-11-16T05:54:36,578 DEBUG [StoreOpener-c81e1c1af980b616d932866cdd4ff915-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:36,579 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a6c8ea5676b855dac5a939ace6afadb0/.tmp/colfamily11/0e05f6b86cbb47f083d3d1e4d9773bdd is 52, key is sss/colfamily11:colfamily11/1731736292066/Put/seqid=0 2024-11-16T05:54:36,582 INFO [StoreOpener-c81e1c1af980b616d932866cdd4ff915-1 {}] regionserver.HStore(327): Store=c81e1c1af980b616d932866cdd4ff915/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:36,582 INFO [StoreOpener-c81e1c1af980b616d932866cdd4ff915-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region c81e1c1af980b616d932866cdd4ff915 2024-11-16T05:54:36,584 INFO [StoreOpener-c81e1c1af980b616d932866cdd4ff915-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c81e1c1af980b616d932866cdd4ff915 columnFamilyName colfamily31 2024-11-16T05:54:36,584 DEBUG [StoreOpener-c81e1c1af980b616d932866cdd4ff915-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T05:54:36,585 INFO [StoreOpener-c81e1c1af980b616d932866cdd4ff915-1 {}] regionserver.HStore(327): Store=c81e1c1af980b616d932866cdd4ff915/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T05:54:36,585 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for c81e1c1af980b616d932866cdd4ff915 2024-11-16T05:54:36,586 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c81e1c1af980b616d932866cdd4ff915 2024-11-16T05:54:36,589 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c81e1c1af980b616d932866cdd4ff915 2024-11-16T05:54:36,594 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c81e1c1af980b616d932866cdd4ff915/recovered.edits/0000000000000000004 2024-11-16T05:54:36,603 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c81e1c1af980b616d932866cdd4ff915/recovered.edits/0000000000000000004 2024-11-16T05:54:36,603 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(2902): Flushing c81e1c1af980b616d932866cdd4ff915 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T05:54:36,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741883_1059 (size=39263) 2024-11-16T05:54:36,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741883_1059 (size=39263) 2024-11-16T05:54:36,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741883_1059 (size=39263) 2024-11-16T05:54:36,630 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a6c8ea5676b855dac5a939ace6afadb0/.tmp/colfamily11/0e05f6b86cbb47f083d3d1e4d9773bdd 2024-11-16T05:54:36,635 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c81e1c1af980b616d932866cdd4ff915/.tmp/colfamily11/cd243e6bc0ba4b47af60fccc88a90d5c is 52, key is iii/colfamily11:colfamily11/1731736291952/Put/seqid=0 2024-11-16T05:54:36,641 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a6c8ea5676b855dac5a939ace6afadb0/.tmp/colfamily11/0e05f6b86cbb47f083d3d1e4d9773bdd as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a6c8ea5676b855dac5a939ace6afadb0/colfamily11/0e05f6b86cbb47f083d3d1e4d9773bdd 2024-11-16T05:54:36,662 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a6c8ea5676b855dac5a939ace6afadb0/colfamily11/0e05f6b86cbb47f083d3d1e4d9773bdd, entries=703, sequenceid=4, filesize=38.3 K 2024-11-16T05:54:36,663 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for a6c8ea5676b855dac5a939ace6afadb0 in 109ms, sequenceid=4, compaction requested=false; wal=null 2024-11-16T05:54:36,664 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a6c8ea5676b855dac5a939ace6afadb0/recovered.edits/0000000000000000004 2024-11-16T05:54:36,666 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for a6c8ea5676b855dac5a939ace6afadb0 2024-11-16T05:54:36,666 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for a6c8ea5676b855dac5a939ace6afadb0 2024-11-16T05:54:36,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741884_1060 (size=39263) 2024-11-16T05:54:36,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741884_1060 (size=39263) 2024-11-16T05:54:36,667 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:54:36,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741884_1060 (size=39263) 2024-11-16T05:54:36,670 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for a6c8ea5676b855dac5a939ace6afadb0 2024-11-16T05:54:36,673 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c81e1c1af980b616d932866cdd4ff915/.tmp/colfamily11/cd243e6bc0ba4b47af60fccc88a90d5c 2024-11-16T05:54:36,686 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c81e1c1af980b616d932866cdd4ff915/.tmp/colfamily11/cd243e6bc0ba4b47af60fccc88a90d5c as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c81e1c1af980b616d932866cdd4ff915/colfamily11/cd243e6bc0ba4b47af60fccc88a90d5c 2024-11-16T05:54:36,693 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a6c8ea5676b855dac5a939ace6afadb0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:54:36,695 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened a6c8ea5676b855dac5a939ace6afadb0; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74546041, jitterRate=0.11082257330417633}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:54:36,695 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a6c8ea5676b855dac5a939ace6afadb0 2024-11-16T05:54:36,695 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for a6c8ea5676b855dac5a939ace6afadb0: Running coprocessor pre-open hook at 1731736476534Writing region info on filesystem at 1731736476534Initializing all the Stores at 1731736476536 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736476536Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736476536Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736476536Obtaining lock to block concurrent updates at 1731736476554 (+18 ms)Preparing flush snapshotting stores in a6c8ea5676b855dac5a939ace6afadb0 at 1731736476554Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731736476554Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0. at 1731736476554Flushing a6c8ea5676b855dac5a939ace6afadb0/colfamily11: creating writer at 1731736476554Flushing a6c8ea5676b855dac5a939ace6afadb0/colfamily11: appending metadata at 1731736476579 (+25 ms)Flushing a6c8ea5676b855dac5a939ace6afadb0/colfamily11: closing flushed file at 1731736476579Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e073ef4: reopening flushed file at 1731736476638 (+59 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for a6c8ea5676b855dac5a939ace6afadb0 in 109ms, sequenceid=4, compaction requested=false; wal=null at 1731736476663 (+25 ms)Cleaning up temporary data from old regions at 1731736476666 (+3 ms)Running coprocessor post-open hooks at 1731736476695 (+29 ms)Region opened successfully at 1731736476695 2024-11-16T05:54:36,696 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0., pid=76, masterSystemTime=1731736476167 2024-11-16T05:54:36,699 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0. 2024-11-16T05:54:36,699 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0. 2024-11-16T05:54:36,700 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c81e1c1af980b616d932866cdd4ff915/colfamily11/cd243e6bc0ba4b47af60fccc88a90d5c, entries=703, sequenceid=4, filesize=38.3 K 2024-11-16T05:54:36,700 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for c81e1c1af980b616d932866cdd4ff915 in 97ms, sequenceid=4, compaction requested=false; wal=null 2024-11-16T05:54:36,700 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=a6c8ea5676b855dac5a939ace6afadb0, regionState=OPEN, openSeqNum=5, regionLocation=3456ee6a3164,33007,1731736285465 2024-11-16T05:54:36,702 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c81e1c1af980b616d932866cdd4ff915/recovered.edits/0000000000000000004 2024-11-16T05:54:36,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=76, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure a6c8ea5676b855dac5a939ace6afadb0, server=3456ee6a3164,33007,1731736285465 because future has completed 2024-11-16T05:54:36,704 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for c81e1c1af980b616d932866cdd4ff915 2024-11-16T05:54:36,705 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for c81e1c1af980b616d932866cdd4ff915 2024-11-16T05:54:36,706 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-16T05:54:36,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=70 2024-11-16T05:54:36,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure a6c8ea5676b855dac5a939ace6afadb0, server=3456ee6a3164,33007,1731736285465 in 683 msec 2024-11-16T05:54:36,710 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for c81e1c1af980b616d932866cdd4ff915 2024-11-16T05:54:36,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a6c8ea5676b855dac5a939ace6afadb0, ASSIGN in 857 msec 2024-11-16T05:54:36,718 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c81e1c1af980b616d932866cdd4ff915/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T05:54:36,720 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened c81e1c1af980b616d932866cdd4ff915; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62369690, jitterRate=-0.07061919569969177}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-16T05:54:36,720 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c81e1c1af980b616d932866cdd4ff915 2024-11-16T05:54:36,720 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for c81e1c1af980b616d932866cdd4ff915: Running coprocessor pre-open hook at 1731736476560Writing region info on filesystem at 1731736476560Initializing all the Stores at 1731736476562 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736476562Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736476569 (+7 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731736476569Obtaining lock to block concurrent updates at 1731736476603 (+34 ms)Preparing flush snapshotting stores in c81e1c1af980b616d932866cdd4ff915 at 1731736476603Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731736476605 (+2 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915. at 1731736476605Flushing c81e1c1af980b616d932866cdd4ff915/colfamily11: creating writer at 1731736476605Flushing c81e1c1af980b616d932866cdd4ff915/colfamily11: appending metadata at 1731736476634 (+29 ms)Flushing c81e1c1af980b616d932866cdd4ff915/colfamily11: closing flushed file at 1731736476634Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e348b60: reopening flushed file at 1731736476684 (+50 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for c81e1c1af980b616d932866cdd4ff915 in 97ms, sequenceid=4, compaction requested=false; wal=null at 1731736476700 (+16 ms)Cleaning up temporary data from old regions at 1731736476705 (+5 ms)Running coprocessor post-open hooks at 1731736476720 (+15 ms)Region opened successfully at 1731736476720 2024-11-16T05:54:36,721 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915., pid=78, masterSystemTime=1731736476163 2024-11-16T05:54:36,724 DEBUG [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915. 2024-11-16T05:54:36,725 INFO [RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915. 2024-11-16T05:54:36,725 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=c81e1c1af980b616d932866cdd4ff915, regionState=OPEN, openSeqNum=5, regionLocation=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:36,727 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c81e1c1af980b616d932866cdd4ff915, server=3456ee6a3164,43415,1731736285705 because future has completed 2024-11-16T05:54:36,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=64 2024-11-16T05:54:36,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure c81e1c1af980b616d932866cdd4ff915, server=3456ee6a3164,43415,1731736285705 in 703 msec 2024-11-16T05:54:36,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=57 2024-11-16T05:54:36,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c81e1c1af980b616d932866cdd4ff915, ASSIGN in 883 msec 2024-11-16T05:54:36,736 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 3456ee6a3164,44781,1731736285630 after splitting done 2024-11-16T05:54:36,736 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-16T05:54:36,736 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=SUCCESS, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true 2024-11-16T05:54:36,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:36,739 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=SUCCESS, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T05:54:36,850 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 8 exec state=pid=57, state=SUCCESS, hasLock=true; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true 2024-11-16T05:54:36,850 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-16T05:54:36,850 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:36,850 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:36,850 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:36,851 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-16T05:54:36,851 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-16T05:54:36,851 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-16T05:54:36,851 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T05:54:36,852 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T05:54:36,854 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-16T05:54:36,854 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:36,854 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:36,855 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:36,855 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:36,855 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:36,855 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:36,855 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:36,855 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:36,855 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:36,855 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:36,855 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:36,855 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:36,855 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:36,855 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:36,855 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:36,855 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:36,855 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:36,855 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:36,855 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-16T05:54:36,855 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-16T05:54:36,855 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-16T05:54:36,855 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-16T05:54:36,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T05:54:36,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-16T05:54:36,859 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_FINISH, hasLock=false; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true 2024-11-16T05:54:36,859 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:36,859 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:36,859 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T05:54:36,859 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:36,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C44781%2C1731736285630.1731736287193, worker=3456ee6a3164,33007,1731736285465 2024-11-16T05:54:36,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C44781%2C1731736285630.1731736287193, worker=3456ee6a3164,33007,1731736285465 2024-11-16T05:54:36,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c81e1c1af980b616d932866cdd4ff915, ASSIGN 2024-11-16T05:54:36,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=883c672d1e7de6e08e15c0c42278e66d, ASSIGN 2024-11-16T05:54:36,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ac83f7461cc15e741d0440c6ee2781db, ASSIGN 2024-11-16T05:54:36,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ec037e054803eda72bad2d81c6143b00, ASSIGN 2024-11-16T05:54:36,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=42e678de61063128349f238929e3d296, ASSIGN 2024-11-16T05:54:36,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bd79de8195c0452a334d7da789e8f23b, ASSIGN 2024-11-16T05:54:36,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a6c8ea5676b855dac5a939ace6afadb0, ASSIGN 2024-11-16T05:54:36,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=10a3e438f5882d261cd799e4d6b75095, ASSIGN 2024-11-16T05:54:36,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=71, state=SUCCESS, hasLock=false; OpenRegionProcedure 10a3e438f5882d261cd799e4d6b75095, server=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:36,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 42e678de61063128349f238929e3d296, server=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:36,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure bd79de8195c0452a334d7da789e8f23b, server=3456ee6a3164,33007,1731736285465 2024-11-16T05:54:36,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=65, state=SUCCESS, hasLock=false; OpenRegionProcedure 883c672d1e7de6e08e15c0c42278e66d, server=3456ee6a3164,33007,1731736285465 2024-11-16T05:54:36,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure a6c8ea5676b855dac5a939ace6afadb0, server=3456ee6a3164,33007,1731736285465 2024-11-16T05:54:36,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure ac83f7461cc15e741d0440c6ee2781db, server=3456ee6a3164,33007,1731736285465 2024-11-16T05:54:36,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure c81e1c1af980b616d932866cdd4ff915, server=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:36,860 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure ec037e054803eda72bad2d81c6143b00, server=3456ee6a3164,43415,1731736285705 2024-11-16T05:54:36,860 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_FINISH, hasLock=false; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-16T05:54:36,860 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:36,860 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C44781%2C1731736285630.meta.1731736287669.meta, worker=3456ee6a3164,43415,1731736285705 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:36,860 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:36,860 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3456ee6a3164,43415,1731736285705 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:36,860 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3456ee6a3164%2C44781%2C1731736285630.1731736287193, worker=3456ee6a3164,33007,1731736285465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:36,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3456ee6a3164%2C44781%2C1731736285630.1731736287193, worker=3456ee6a3164,33007,1731736285465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:36,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=c81e1c1af980b616d932866cdd4ff915, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:36,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=883c672d1e7de6e08e15c0c42278e66d, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:36,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ac83f7461cc15e741d0440c6ee2781db, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:36,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=67, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ec037e054803eda72bad2d81c6143b00, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:36,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=68, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=42e678de61063128349f238929e3d296, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:36,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=69, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bd79de8195c0452a334d7da789e8f23b, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:36,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=70, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a6c8ea5676b855dac5a939ace6afadb0, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:36,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=71, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=10a3e438f5882d261cd799e4d6b75095, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:36,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=71, state=SUCCESS, hasLock=false; OpenRegionProcedure 10a3e438f5882d261cd799e4d6b75095, server=3456ee6a3164,43415,1731736285705 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:36,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 42e678de61063128349f238929e3d296, server=3456ee6a3164,43415,1731736285705 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:36,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure bd79de8195c0452a334d7da789e8f23b, server=3456ee6a3164,33007,1731736285465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:36,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=65, state=SUCCESS, hasLock=false; OpenRegionProcedure 883c672d1e7de6e08e15c0c42278e66d, server=3456ee6a3164,33007,1731736285465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:36,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure a6c8ea5676b855dac5a939ace6afadb0, server=3456ee6a3164,33007,1731736285465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:36,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure ac83f7461cc15e741d0440c6ee2781db, server=3456ee6a3164,33007,1731736285465 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:36,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure c81e1c1af980b616d932866cdd4ff915, server=3456ee6a3164,43415,1731736285705 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:36,861 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure ec037e054803eda72bad2d81c6143b00, server=3456ee6a3164,43415,1731736285705 didn't hold the lock before restarting, skip acquiring lock. 2024-11-16T05:54:36,861 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-16T05:54:36,862 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=61 2024-11-16T05:54:36,862 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3456ee6a3164,43415,1731736285705, table=hbase:meta, region=1588230740 2024-11-16T05:54:36,862 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3456ee6a3164,43415,1731736285705, state=OPEN 2024-11-16T05:54:36,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:36,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:36,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T05:54:36,864 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:36,864 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:36,864 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T05:54:36,866 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731736506866 2024-11-16T05:54:36,866 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T05:54:36,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b59301af2f6042086ee9455ab281e7e6, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:36,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=54bd8d95e5df76bdd2472b3a67cd2b8b, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:36,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=de75e055385b7b1b204f0aa9c1924d0e, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:36,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=42e678de61063128349f238929e3d296, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=5 2024-11-16T05:54:36,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=53faee5a89035a5faf9a235d93b486c9, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:36,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3e30199c6045096f8f565c93af9b6088, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:36,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=883c672d1e7de6e08e15c0c42278e66d, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=5 2024-11-16T05:54:36,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ff61a3918e3c512081ba7b9c1e1a4948, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:36,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c81e1c1af980b616d932866cdd4ff915, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=5 2024-11-16T05:54:36,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5baf9d0bdd8c65c757ad796f93bef122, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:36,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=63baab6432268e686bc2f01a98a6c75f, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:36,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=408fc7812ce9cee6ea99428f155ce5a8, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:36,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bd79de8195c0452a334d7da789e8f23b, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=5 2024-11-16T05:54:36,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12c6ab39e16284ff5b8ffeb36e672416, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:36,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=07f1b4025bc2e2627b7dc3f7c79afd8e, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:36,882 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ec037e054803eda72bad2d81c6143b00, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=5 2024-11-16T05:54:36,883 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0166203610c1853876df2910d136e997, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:36,883 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df4b7c945ebb852a560a10e20c40c386, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:36,883 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a6c8ea5676b855dac5a939ace6afadb0, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=5 2024-11-16T05:54:36,883 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2beaf8caee4e552158f6bb091a0437bb, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:36,883 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9df9b8a2b43462b728e2352478a97347, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:36,883 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=10a3e438f5882d261cd799e4d6b75095, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=5 2024-11-16T05:54:36,883 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb73dbcec883aba378443bd58f1eb724, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:36,883 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac83f7461cc15e741d0440c6ee2781db, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=5 2024-11-16T05:54:36,884 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=afd4d1a884eb847b35b730a1b7a90e0d, regionState=OPEN, lastHost=3456ee6a3164,43415,1731736285705, regionLocation=3456ee6a3164,43415,1731736285705, openSeqNum=2 2024-11-16T05:54:36,884 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ca14297d921e6ca3fee0822ad2bb732e, regionState=OPEN, lastHost=3456ee6a3164,33007,1731736285465, regionLocation=3456ee6a3164,33007,1731736285465, openSeqNum=2 2024-11-16T05:54:36,884 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T05:54:36,884 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731736536884 2024-11-16T05:54:36,884 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731736596884 2024-11-16T05:54:36,884 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 17 msec 2024-11-16T05:54:36,884 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 3456ee6a3164,44781,1731736285630 after splitting done 2024-11-16T05:54:36,884 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-16T05:54:36,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; ServerCrashProcedure 3456ee6a3164,44781,1731736285630, splitWal=true, meta=true in 3 mins, 4.03 sec 2024-11-16T05:54:37,138 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40414, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T05:54:37,141 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:54:37,149 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49066, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T05:54:37,162 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3456ee6a3164:44781 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 3456ee6a3164/172.17.0.2:44781 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T05:54:37,163 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296., hostname=3456ee6a3164,44781,1731736285630, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296., hostname=3456ee6a3164,44781,1731736285630, seqNum=2, error=java.net.ConnectException: Call to address=3456ee6a3164:44781 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:54:37,164 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296., hostname=3456ee6a3164,44781,1731736285630, seqNum=2 is java.net.ConnectException: Connection refused 2024-11-16T05:54:37,164 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296., hostname=3456ee6a3164,44781,1731736285630, seqNum=2 from cache 2024-11-16T05:54:37,164 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address 3456ee6a3164:44781 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 3456ee6a3164/172.17.0.2:44781 2024-11-16T05:54:37,275 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3456ee6a3164:44781 this server is in the failed servers list 2024-11-16T05:54:37,275 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:54:37,275 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:54:37,275 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,44781,1731736285630, seqNum=-1 from cache 2024-11-16T05:54:37,384 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T05:54:37,385 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40422, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T05:54:37,386 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3456ee6a3164,43415,1731736285705, seqNum=-1] 2024-11-16T05:54:37,391 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296., hostname=3456ee6a3164,43415,1731736285705, seqNum=5] 2024-11-16T05:54:37,447 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3456ee6a3164:44781 this server is in the failed servers list 2024-11-16T05:54:37,448 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d., hostname=3456ee6a3164,44781,1731736285630, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d., hostname=3456ee6a3164,44781,1731736285630, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:54:37,448 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d., hostname=3456ee6a3164,44781,1731736285630, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:54:37,448 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d., hostname=3456ee6a3164,44781,1731736285630, seqNum=2 from cache 2024-11-16T05:54:37,565 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d., hostname=3456ee6a3164,33007,1731736285465, seqNum=5] 2024-11-16T05:54:37,601 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3456ee6a3164:44781 this server is in the failed servers list 2024-11-16T05:54:37,602 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915., hostname=3456ee6a3164,44781,1731736285630, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915., hostname=3456ee6a3164,44781,1731736285630, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:54:37,602 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915., hostname=3456ee6a3164,44781,1731736285630, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:54:37,602 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915., hostname=3456ee6a3164,44781,1731736285630, seqNum=2 from cache 2024-11-16T05:54:37,710 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iii', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915., hostname=3456ee6a3164,43415,1731736285705, seqNum=5] 2024-11-16T05:54:37,767 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3456ee6a3164:44781 this server is in the failed servers list 2024-11-16T05:54:37,767 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b., hostname=3456ee6a3164,44781,1731736285630, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b., hostname=3456ee6a3164,44781,1731736285630, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:54:37,768 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b., hostname=3456ee6a3164,44781,1731736285630, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:54:37,768 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b., hostname=3456ee6a3164,44781,1731736285630, seqNum=2 from cache 2024-11-16T05:54:37,882 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b., hostname=3456ee6a3164,33007,1731736285465, seqNum=5] 2024-11-16T05:54:37,922 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3456ee6a3164:44781 this server is in the failed servers list 2024-11-16T05:54:37,922 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00., hostname=3456ee6a3164,44781,1731736285630, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00., hostname=3456ee6a3164,44781,1731736285630, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:54:37,922 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00., hostname=3456ee6a3164,44781,1731736285630, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:54:37,922 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00., hostname=3456ee6a3164,44781,1731736285630, seqNum=2 from cache 2024-11-16T05:54:38,031 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00., hostname=3456ee6a3164,43415,1731736285705, seqNum=5] 2024-11-16T05:54:38,052 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3456ee6a3164:44781 this server is in the failed servers list 2024-11-16T05:54:38,053 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0., hostname=3456ee6a3164,44781,1731736285630, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0., hostname=3456ee6a3164,44781,1731736285630, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:54:38,053 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0., hostname=3456ee6a3164,44781,1731736285630, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:54:38,053 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0., hostname=3456ee6a3164,44781,1731736285630, seqNum=2 from cache 2024-11-16T05:54:38,168 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='sss', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0., hostname=3456ee6a3164,33007,1731736285465, seqNum=5] 2024-11-16T05:54:38,183 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3456ee6a3164:44781 this server is in the failed servers list 2024-11-16T05:54:38,184 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095., hostname=3456ee6a3164,44781,1731736285630, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095., hostname=3456ee6a3164,44781,1731736285630, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:54:38,184 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095., hostname=3456ee6a3164,44781,1731736285630, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:54:38,184 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095., hostname=3456ee6a3164,44781,1731736285630, seqNum=2 from cache 2024-11-16T05:54:38,299 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095., hostname=3456ee6a3164,43415,1731736285705, seqNum=5] 2024-11-16T05:54:38,312 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3456ee6a3164:44781 this server is in the failed servers list 2024-11-16T05:54:38,312 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db., hostname=3456ee6a3164,44781,1731736285630, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db., hostname=3456ee6a3164,44781,1731736285630, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3456ee6a3164:44781 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:54:38,312 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db., hostname=3456ee6a3164,44781,1731736285630, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3456ee6a3164:44781 2024-11-16T05:54:38,313 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db., hostname=3456ee6a3164,44781,1731736285630, seqNum=2 from cache 2024-11-16T05:54:38,430 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db., hostname=3456ee6a3164,33007,1731736285465, seqNum=5] 2024-11-16T05:54:38,551 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-16T05:54:38,551 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-16T05:54:38,551 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T05:54:38,551 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.procedure.TestSCPBase.tearDown(TestSCPBase.java:72) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:54:38,551 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:54:38,552 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:54:38,552 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T05:54:38,553 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-16T05:54:38,553 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=534022733, stopped=false 2024-11-16T05:54:38,553 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=3456ee6a3164,44685,1731736284655 2024-11-16T05:54:38,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T05:54:38,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T05:54:38,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T05:54:38,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:38,557 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:38,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:38,557 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T05:54:38,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:38,557 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.procedure.TestSCPBase.tearDown(TestSCPBase.java:72) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:54:38,557 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:54:38,558 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3456ee6a3164,33007,1731736285465' ***** 2024-11-16T05:54:38,558 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T05:54:38,558 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:54:38,558 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3456ee6a3164,43415,1731736285705' ***** 2024-11-16T05:54:38,558 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T05:54:38,558 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:54:38,558 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T05:54:38,558 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T05:54:38,559 INFO [RS:2;3456ee6a3164:43415 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T05:54:38,559 INFO [RS:2;3456ee6a3164:43415 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T05:54:38,559 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T05:54:38,559 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(3091): Received CLOSE for 5baf9d0bdd8c65c757ad796f93bef122 2024-11-16T05:54:38,559 INFO [RS:0;3456ee6a3164:33007 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T05:54:38,559 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T05:54:38,559 INFO [RS:0;3456ee6a3164:33007 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T05:54:38,559 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer(3091): Received CLOSE for a6c8ea5676b855dac5a939ace6afadb0 2024-11-16T05:54:38,559 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T05:54:38,559 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer(3091): Received CLOSE for b59301af2f6042086ee9455ab281e7e6 2024-11-16T05:54:38,559 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer(3091): Received CLOSE for 12c6ab39e16284ff5b8ffeb36e672416 2024-11-16T05:54:38,559 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(3091): Received CLOSE for c81e1c1af980b616d932866cdd4ff915 2024-11-16T05:54:38,559 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer(3091): Received CLOSE for 9df9b8a2b43462b728e2352478a97347 2024-11-16T05:54:38,559 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(3091): Received CLOSE for 54bd8d95e5df76bdd2472b3a67cd2b8b 2024-11-16T05:54:38,559 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer(3091): Received CLOSE for 63baab6432268e686bc2f01a98a6c75f 2024-11-16T05:54:38,559 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(3091): Received CLOSE for ec037e054803eda72bad2d81c6143b00 2024-11-16T05:54:38,559 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer(3091): Received CLOSE for 883c672d1e7de6e08e15c0c42278e66d 2024-11-16T05:54:38,559 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(3091): Received CLOSE for 07f1b4025bc2e2627b7dc3f7c79afd8e 2024-11-16T05:54:38,560 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer(3091): Received CLOSE for ac83f7461cc15e741d0440c6ee2781db 2024-11-16T05:54:38,560 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(3091): Received CLOSE for df4b7c945ebb852a560a10e20c40c386 2024-11-16T05:54:38,560 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer(3091): Received CLOSE for ca14297d921e6ca3fee0822ad2bb732e 2024-11-16T05:54:38,560 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(3091): Received CLOSE for 53faee5a89035a5faf9a235d93b486c9 2024-11-16T05:54:38,560 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer(3091): Received CLOSE for 0166203610c1853876df2910d136e997 2024-11-16T05:54:38,560 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(3091): Received CLOSE for 408fc7812ce9cee6ea99428f155ce5a8 2024-11-16T05:54:38,560 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer(3091): Received CLOSE for 3e30199c6045096f8f565c93af9b6088 2024-11-16T05:54:38,560 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(3091): Received CLOSE for ff61a3918e3c512081ba7b9c1e1a4948 2024-11-16T05:54:38,560 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer(3091): Received CLOSE for eb73dbcec883aba378443bd58f1eb724 2024-11-16T05:54:38,560 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(3091): Received CLOSE for 10a3e438f5882d261cd799e4d6b75095 2024-11-16T05:54:38,560 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer(3091): Received CLOSE for bd79de8195c0452a334d7da789e8f23b 2024-11-16T05:54:38,560 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer(3091): Received CLOSE for de75e055385b7b1b204f0aa9c1924d0e 2024-11-16T05:54:38,560 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(3091): Received CLOSE for 42e678de61063128349f238929e3d296 2024-11-16T05:54:38,560 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(3091): Received CLOSE for 2beaf8caee4e552158f6bb091a0437bb 2024-11-16T05:54:38,560 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a6c8ea5676b855dac5a939ace6afadb0, disabling compactions & flushes 2024-11-16T05:54:38,560 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(3091): Received CLOSE for afd4d1a884eb847b35b730a1b7a90e0d 2024-11-16T05:54:38,560 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer(959): stopping server 3456ee6a3164,33007,1731736285465 2024-11-16T05:54:38,560 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0. 2024-11-16T05:54:38,560 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(959): stopping server 3456ee6a3164,43415,1731736285705 2024-11-16T05:54:38,560 INFO [RS:0;3456ee6a3164:33007 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T05:54:38,560 INFO [RS:2;3456ee6a3164:43415 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T05:54:38,560 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0. 2024-11-16T05:54:38,560 INFO [RS:0;3456ee6a3164:33007 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;3456ee6a3164:33007. 2024-11-16T05:54:38,560 INFO [RS:2;3456ee6a3164:43415 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;3456ee6a3164:43415. 2024-11-16T05:54:38,560 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0. after waiting 0 ms 2024-11-16T05:54:38,560 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0. 2024-11-16T05:54:38,560 DEBUG [RS:0;3456ee6a3164:33007 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:54:38,560 DEBUG [RS:2;3456ee6a3164:43415 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T05:54:38,560 DEBUG [RS:0;3456ee6a3164:33007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:54:38,560 DEBUG [RS:2;3456ee6a3164:43415 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:54:38,560 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-16T05:54:38,560 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T05:54:38,560 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T05:54:38,560 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T05:54:38,561 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T05:54:38,561 DEBUG [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer(1351): Waiting on 0166203610c1853876df2910d136e997, 12c6ab39e16284ff5b8ffeb36e672416, 3e30199c6045096f8f565c93af9b6088, 63baab6432268e686bc2f01a98a6c75f, 883c672d1e7de6e08e15c0c42278e66d, 9df9b8a2b43462b728e2352478a97347, a6c8ea5676b855dac5a939ace6afadb0, ac83f7461cc15e741d0440c6ee2781db, b59301af2f6042086ee9455ab281e7e6, bd79de8195c0452a334d7da789e8f23b, ca14297d921e6ca3fee0822ad2bb732e, de75e055385b7b1b204f0aa9c1924d0e, eb73dbcec883aba378443bd58f1eb724 2024-11-16T05:54:38,561 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-11-16T05:54:38,561 DEBUG [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(1351): Waiting on 07f1b4025bc2e2627b7dc3f7c79afd8e, 10a3e438f5882d261cd799e4d6b75095, 1588230740, 2beaf8caee4e552158f6bb091a0437bb, 408fc7812ce9cee6ea99428f155ce5a8, 42e678de61063128349f238929e3d296, 53faee5a89035a5faf9a235d93b486c9, 54bd8d95e5df76bdd2472b3a67cd2b8b, 5baf9d0bdd8c65c757ad796f93bef122, afd4d1a884eb847b35b730a1b7a90e0d, c81e1c1af980b616d932866cdd4ff915, df4b7c945ebb852a560a10e20c40c386, ec037e054803eda72bad2d81c6143b00, ff61a3918e3c512081ba7b9c1e1a4948 2024-11-16T05:54:38,561 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T05:54:38,561 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T05:54:38,561 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T05:54:38,561 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T05:54:38,561 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T05:54:38,562 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=11.91 KB heapSize=17.63 KB 2024-11-16T05:54:38,565 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5baf9d0bdd8c65c757ad796f93bef122, disabling compactions & flushes 2024-11-16T05:54:38,565 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731736288173.5baf9d0bdd8c65c757ad796f93bef122. 2024-11-16T05:54:38,565 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731736288173.5baf9d0bdd8c65c757ad796f93bef122. 2024-11-16T05:54:38,565 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731736288173.5baf9d0bdd8c65c757ad796f93bef122. after waiting 0 ms 2024-11-16T05:54:38,565 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731736288173.5baf9d0bdd8c65c757ad796f93bef122. 2024-11-16T05:54:38,565 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 5baf9d0bdd8c65c757ad796f93bef122 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T05:54:38,575 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/.tmp/info/6557be4c95834cf0a8c5448f945c81c7 is 268, key is testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296./info:regioninfo/1731736476560/Put/seqid=0 2024-11-16T05:54:38,586 INFO [regionserver/3456ee6a3164:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T05:54:38,586 INFO [regionserver/3456ee6a3164:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T05:54:38,593 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a6c8ea5676b855dac5a939ace6afadb0/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-16T05:54:38,597 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5baf9d0bdd8c65c757ad796f93bef122/.tmp/colfamily11/0d8da05dfb784ad2b3387db59fd459a5 is 52, key is jjj/colfamily11:colfamily11/1731736292000/Put/seqid=0 2024-11-16T05:54:38,601 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0. 2024-11-16T05:54:38,602 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a6c8ea5676b855dac5a939ace6afadb0: Waiting for close lock at 1731736478559Running coprocessor pre-close hooks at 1731736478560 (+1 ms)Disabling compacts and flushes for region at 1731736478560Disabling writes for close at 1731736478560Writing region close event to WAL at 1731736478573 (+13 ms)Running coprocessor post-close hooks at 1731736478594 (+21 ms)Closed at 1731736478601 (+7 ms) 2024-11-16T05:54:38,602 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731736288173.a6c8ea5676b855dac5a939ace6afadb0. 2024-11-16T05:54:38,603 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b59301af2f6042086ee9455ab281e7e6, disabling compactions & flushes 2024-11-16T05:54:38,603 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6. 2024-11-16T05:54:38,603 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6. 2024-11-16T05:54:38,603 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6. after waiting 0 ms 2024-11-16T05:54:38,603 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6. 2024-11-16T05:54:38,603 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing b59301af2f6042086ee9455ab281e7e6 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T05:54:38,646 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b59301af2f6042086ee9455ab281e7e6/.tmp/colfamily11/1e0b6a1b4ae7402a85b3477baa460d24 is 52, key is aaa/colfamily11:colfamily11/1731736291779/Put/seqid=0 2024-11-16T05:54:38,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741885_1061 (size=18322) 2024-11-16T05:54:38,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741885_1061 (size=18322) 2024-11-16T05:54:38,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741885_1061 (size=18322) 2024-11-16T05:54:38,666 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.91 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/.tmp/info/6557be4c95834cf0a8c5448f945c81c7 2024-11-16T05:54:38,677 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/.tmp/info/6557be4c95834cf0a8c5448f945c81c7 as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/info/6557be4c95834cf0a8c5448f945c81c7 2024-11-16T05:54:38,689 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/info/6557be4c95834cf0a8c5448f945c81c7, entries=64, sequenceid=79, filesize=17.9 K 2024-11-16T05:54:38,691 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~11.91 KB/12192, heapSize ~16.86 KB/17264, currentSize=0 B/0 for 1588230740 in 130ms, sequenceid=79, compaction requested=false 2024-11-16T05:54:38,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741886_1062 (size=39263) 2024-11-16T05:54:38,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741886_1062 (size=39263) 2024-11-16T05:54:38,701 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5baf9d0bdd8c65c757ad796f93bef122/.tmp/colfamily11/0d8da05dfb784ad2b3387db59fd459a5 2024-11-16T05:54:38,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741886_1062 (size=39263) 2024-11-16T05:54:38,716 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5baf9d0bdd8c65c757ad796f93bef122/.tmp/colfamily11/0d8da05dfb784ad2b3387db59fd459a5 as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5baf9d0bdd8c65c757ad796f93bef122/colfamily11/0d8da05dfb784ad2b3387db59fd459a5 2024-11-16T05:54:38,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741887_1063 (size=39263) 2024-11-16T05:54:38,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741887_1063 (size=39263) 2024-11-16T05:54:38,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741887_1063 (size=39263) 2024-11-16T05:54:38,722 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b59301af2f6042086ee9455ab281e7e6/.tmp/colfamily11/1e0b6a1b4ae7402a85b3477baa460d24 2024-11-16T05:54:38,726 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5baf9d0bdd8c65c757ad796f93bef122/colfamily11/0d8da05dfb784ad2b3387db59fd459a5, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T05:54:38,728 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 5baf9d0bdd8c65c757ad796f93bef122 in 162ms, sequenceid=5, compaction requested=false 2024-11-16T05:54:38,730 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b59301af2f6042086ee9455ab281e7e6/.tmp/colfamily11/1e0b6a1b4ae7402a85b3477baa460d24 as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b59301af2f6042086ee9455ab281e7e6/colfamily11/1e0b6a1b4ae7402a85b3477baa460d24 2024-11-16T05:54:38,738 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/hbase/meta/1588230740/recovered.edits/82.seqid, newMaxSeqId=82, maxSeqId=60 2024-11-16T05:54:38,738 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b59301af2f6042086ee9455ab281e7e6/colfamily11/1e0b6a1b4ae7402a85b3477baa460d24, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T05:54:38,739 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T05:54:38,739 INFO [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T05:54:38,739 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731736478561Running coprocessor pre-close hooks at 1731736478561Disabling compacts and flushes for region at 1731736478561Disabling writes for close at 1731736478561Obtaining lock to block concurrent updates at 1731736478562 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731736478562Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=12192, getHeapSize=17984, getOffHeapSize=0, getCellsCount=64 at 1731736478567 (+5 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731736478569 (+2 ms)Flushing 1588230740/info: creating writer at 1731736478569Flushing 1588230740/info: appending metadata at 1731736478575 (+6 ms)Flushing 1588230740/info: closing flushed file at 1731736478575Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33651992: reopening flushed file at 1731736478676 (+101 ms)Finished flush of dataSize ~11.91 KB/12192, heapSize ~16.86 KB/17264, currentSize=0 B/0 for 1588230740 in 130ms, sequenceid=79, compaction requested=false at 1731736478691 (+15 ms)Writing region close event to WAL at 1731736478729 (+38 ms)Running coprocessor post-close hooks at 1731736478739 (+10 ms)Closed at 1731736478739 2024-11-16T05:54:38,739 DEBUG [RS_CLOSE_META-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T05:54:38,739 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for b59301af2f6042086ee9455ab281e7e6 in 136ms, sequenceid=5, compaction requested=false 2024-11-16T05:54:38,749 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/5baf9d0bdd8c65c757ad796f93bef122/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T05:54:38,750 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731736288173.5baf9d0bdd8c65c757ad796f93bef122. 2024-11-16T05:54:38,750 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5baf9d0bdd8c65c757ad796f93bef122: Waiting for close lock at 1731736478565Running coprocessor pre-close hooks at 1731736478565Disabling compacts and flushes for region at 1731736478565Disabling writes for close at 1731736478565Obtaining lock to block concurrent updates at 1731736478565Preparing flush snapshotting stores in 5baf9d0bdd8c65c757ad796f93bef122 at 1731736478565Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731736288173.5baf9d0bdd8c65c757ad796f93bef122., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731736478567 (+2 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731736288173.5baf9d0bdd8c65c757ad796f93bef122. at 1731736478569 (+2 ms)Flushing 5baf9d0bdd8c65c757ad796f93bef122/colfamily11: creating writer at 1731736478569Flushing 5baf9d0bdd8c65c757ad796f93bef122/colfamily11: appending metadata at 1731736478597 (+28 ms)Flushing 5baf9d0bdd8c65c757ad796f93bef122/colfamily11: closing flushed file at 1731736478597Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14f97828: reopening flushed file at 1731736478714 (+117 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 5baf9d0bdd8c65c757ad796f93bef122 in 162ms, sequenceid=5, compaction requested=false at 1731736478728 (+14 ms)Writing region close event to WAL at 1731736478737 (+9 ms)Running coprocessor post-close hooks at 1731736478750 (+13 ms)Closed at 1731736478750 2024-11-16T05:54:38,750 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731736288173.5baf9d0bdd8c65c757ad796f93bef122. 2024-11-16T05:54:38,751 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c81e1c1af980b616d932866cdd4ff915, disabling compactions & flushes 2024-11-16T05:54:38,751 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915. 2024-11-16T05:54:38,751 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915. 2024-11-16T05:54:38,751 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915. after waiting 0 ms 2024-11-16T05:54:38,751 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915. 2024-11-16T05:54:38,758 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b59301af2f6042086ee9455ab281e7e6/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T05:54:38,761 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6. 2024-11-16T05:54:38,761 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b59301af2f6042086ee9455ab281e7e6: Waiting for close lock at 1731736478603Running coprocessor pre-close hooks at 1731736478603Disabling compacts and flushes for region at 1731736478603Disabling writes for close at 1731736478603Obtaining lock to block concurrent updates at 1731736478603Preparing flush snapshotting stores in b59301af2f6042086ee9455ab281e7e6 at 1731736478603Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731736478604 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6. at 1731736478611 (+7 ms)Flushing b59301af2f6042086ee9455ab281e7e6/colfamily11: creating writer at 1731736478612 (+1 ms)Flushing b59301af2f6042086ee9455ab281e7e6/colfamily11: appending metadata at 1731736478646 (+34 ms)Flushing b59301af2f6042086ee9455ab281e7e6/colfamily11: closing flushed file at 1731736478646Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@243af658: reopening flushed file at 1731736478729 (+83 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for b59301af2f6042086ee9455ab281e7e6 in 136ms, sequenceid=5, compaction requested=false at 1731736478739 (+10 ms)Writing region close event to WAL at 1731736478749 (+10 ms)Running coprocessor post-close hooks at 1731736478760 (+11 ms)Closed at 1731736478760 2024-11-16T05:54:38,761 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731736288173.b59301af2f6042086ee9455ab281e7e6. 2024-11-16T05:54:38,761 DEBUG [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer(1351): Waiting on 0166203610c1853876df2910d136e997, 12c6ab39e16284ff5b8ffeb36e672416, 3e30199c6045096f8f565c93af9b6088, 63baab6432268e686bc2f01a98a6c75f, 883c672d1e7de6e08e15c0c42278e66d, 9df9b8a2b43462b728e2352478a97347, ac83f7461cc15e741d0440c6ee2781db, b59301af2f6042086ee9455ab281e7e6, bd79de8195c0452a334d7da789e8f23b, ca14297d921e6ca3fee0822ad2bb732e, de75e055385b7b1b204f0aa9c1924d0e, eb73dbcec883aba378443bd58f1eb724 2024-11-16T05:54:38,761 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 12c6ab39e16284ff5b8ffeb36e672416, disabling compactions & flushes 2024-11-16T05:54:38,761 DEBUG [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(1351): Waiting on 07f1b4025bc2e2627b7dc3f7c79afd8e, 10a3e438f5882d261cd799e4d6b75095, 2beaf8caee4e552158f6bb091a0437bb, 408fc7812ce9cee6ea99428f155ce5a8, 42e678de61063128349f238929e3d296, 53faee5a89035a5faf9a235d93b486c9, 54bd8d95e5df76bdd2472b3a67cd2b8b, afd4d1a884eb847b35b730a1b7a90e0d, c81e1c1af980b616d932866cdd4ff915, df4b7c945ebb852a560a10e20c40c386, ec037e054803eda72bad2d81c6143b00, ff61a3918e3c512081ba7b9c1e1a4948 2024-11-16T05:54:38,761 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731736288173.12c6ab39e16284ff5b8ffeb36e672416. 2024-11-16T05:54:38,761 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731736288173.12c6ab39e16284ff5b8ffeb36e672416. 2024-11-16T05:54:38,761 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731736288173.12c6ab39e16284ff5b8ffeb36e672416. after waiting 0 ms 2024-11-16T05:54:38,761 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731736288173.12c6ab39e16284ff5b8ffeb36e672416. 2024-11-16T05:54:38,761 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 12c6ab39e16284ff5b8ffeb36e672416 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T05:54:38,768 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/c81e1c1af980b616d932866cdd4ff915/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-16T05:54:38,768 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915. 2024-11-16T05:54:38,768 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c81e1c1af980b616d932866cdd4ff915: Waiting for close lock at 1731736478751Running coprocessor pre-close hooks at 1731736478751Disabling compacts and flushes for region at 1731736478751Disabling writes for close at 1731736478751Writing region close event to WAL at 1731736478757 (+6 ms)Running coprocessor post-close hooks at 1731736478768 (+11 ms)Closed at 1731736478768 2024-11-16T05:54:38,769 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731736288173.c81e1c1af980b616d932866cdd4ff915. 2024-11-16T05:54:38,769 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 54bd8d95e5df76bdd2472b3a67cd2b8b, disabling compactions & flushes 2024-11-16T05:54:38,769 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731736288173.54bd8d95e5df76bdd2472b3a67cd2b8b. 2024-11-16T05:54:38,769 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731736288173.54bd8d95e5df76bdd2472b3a67cd2b8b. 2024-11-16T05:54:38,769 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731736288173.54bd8d95e5df76bdd2472b3a67cd2b8b. after waiting 0 ms 2024-11-16T05:54:38,769 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731736288173.54bd8d95e5df76bdd2472b3a67cd2b8b. 2024-11-16T05:54:38,769 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 54bd8d95e5df76bdd2472b3a67cd2b8b 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T05:54:38,789 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/12c6ab39e16284ff5b8ffeb36e672416/.tmp/colfamily11/fd58fcfea57d431289091d6ee2ba1509 is 52, key is nnn/colfamily11:colfamily11/1731736292018/Put/seqid=0 2024-11-16T05:54:38,799 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/54bd8d95e5df76bdd2472b3a67cd2b8b/.tmp/colfamily11/2d23f792853e4b8b8acd5417b03a7b5d is 52, key is bbb/colfamily11:colfamily11/1731736291786/Put/seqid=0 2024-11-16T05:54:38,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741888_1064 (size=39263) 2024-11-16T05:54:38,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741888_1064 (size=39263) 2024-11-16T05:54:38,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741888_1064 (size=39263) 2024-11-16T05:54:38,808 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/12c6ab39e16284ff5b8ffeb36e672416/.tmp/colfamily11/fd58fcfea57d431289091d6ee2ba1509 2024-11-16T05:54:38,817 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/12c6ab39e16284ff5b8ffeb36e672416/.tmp/colfamily11/fd58fcfea57d431289091d6ee2ba1509 as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/12c6ab39e16284ff5b8ffeb36e672416/colfamily11/fd58fcfea57d431289091d6ee2ba1509 2024-11-16T05:54:38,834 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/12c6ab39e16284ff5b8ffeb36e672416/colfamily11/fd58fcfea57d431289091d6ee2ba1509, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T05:54:38,835 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 12c6ab39e16284ff5b8ffeb36e672416 in 74ms, sequenceid=5, compaction requested=false 2024-11-16T05:54:38,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741889_1065 (size=39263) 2024-11-16T05:54:38,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741889_1065 (size=39263) 2024-11-16T05:54:38,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741889_1065 (size=39263) 2024-11-16T05:54:38,846 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/54bd8d95e5df76bdd2472b3a67cd2b8b/.tmp/colfamily11/2d23f792853e4b8b8acd5417b03a7b5d 2024-11-16T05:54:38,858 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/54bd8d95e5df76bdd2472b3a67cd2b8b/.tmp/colfamily11/2d23f792853e4b8b8acd5417b03a7b5d as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/54bd8d95e5df76bdd2472b3a67cd2b8b/colfamily11/2d23f792853e4b8b8acd5417b03a7b5d 2024-11-16T05:54:38,858 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/12c6ab39e16284ff5b8ffeb36e672416/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T05:54:38,859 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731736288173.12c6ab39e16284ff5b8ffeb36e672416. 2024-11-16T05:54:38,859 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 12c6ab39e16284ff5b8ffeb36e672416: Waiting for close lock at 1731736478761Running coprocessor pre-close hooks at 1731736478761Disabling compacts and flushes for region at 1731736478761Disabling writes for close at 1731736478761Obtaining lock to block concurrent updates at 1731736478761Preparing flush snapshotting stores in 12c6ab39e16284ff5b8ffeb36e672416 at 1731736478761Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731736288173.12c6ab39e16284ff5b8ffeb36e672416., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731736478762 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731736288173.12c6ab39e16284ff5b8ffeb36e672416. at 1731736478762Flushing 12c6ab39e16284ff5b8ffeb36e672416/colfamily11: creating writer at 1731736478763 (+1 ms)Flushing 12c6ab39e16284ff5b8ffeb36e672416/colfamily11: appending metadata at 1731736478788 (+25 ms)Flushing 12c6ab39e16284ff5b8ffeb36e672416/colfamily11: closing flushed file at 1731736478788Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a1759ca: reopening flushed file at 1731736478816 (+28 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 12c6ab39e16284ff5b8ffeb36e672416 in 74ms, sequenceid=5, compaction requested=false at 1731736478835 (+19 ms)Writing region close event to WAL at 1731736478849 (+14 ms)Running coprocessor post-close hooks at 1731736478859 (+10 ms)Closed at 1731736478859 2024-11-16T05:54:38,860 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731736288173.12c6ab39e16284ff5b8ffeb36e672416. 2024-11-16T05:54:38,860 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9df9b8a2b43462b728e2352478a97347, disabling compactions & flushes 2024-11-16T05:54:38,860 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731736288173.9df9b8a2b43462b728e2352478a97347. 2024-11-16T05:54:38,860 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731736288173.9df9b8a2b43462b728e2352478a97347. 2024-11-16T05:54:38,860 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731736288173.9df9b8a2b43462b728e2352478a97347. after waiting 0 ms 2024-11-16T05:54:38,860 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731736288173.9df9b8a2b43462b728e2352478a97347. 2024-11-16T05:54:38,860 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 9df9b8a2b43462b728e2352478a97347 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T05:54:38,872 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/54bd8d95e5df76bdd2472b3a67cd2b8b/colfamily11/2d23f792853e4b8b8acd5417b03a7b5d, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T05:54:38,880 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 54bd8d95e5df76bdd2472b3a67cd2b8b in 111ms, sequenceid=5, compaction requested=false 2024-11-16T05:54:38,888 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9df9b8a2b43462b728e2352478a97347/.tmp/colfamily11/3376644a07d547999f275c36b0e6cf73 is 52, key is uuu/colfamily11:colfamily11/1731736292098/Put/seqid=0 2024-11-16T05:54:38,900 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/54bd8d95e5df76bdd2472b3a67cd2b8b/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T05:54:38,900 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731736288173.54bd8d95e5df76bdd2472b3a67cd2b8b. 2024-11-16T05:54:38,900 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 54bd8d95e5df76bdd2472b3a67cd2b8b: Waiting for close lock at 1731736478769Running coprocessor pre-close hooks at 1731736478769Disabling compacts and flushes for region at 1731736478769Disabling writes for close at 1731736478769Obtaining lock to block concurrent updates at 1731736478769Preparing flush snapshotting stores in 54bd8d95e5df76bdd2472b3a67cd2b8b at 1731736478769Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731736288173.54bd8d95e5df76bdd2472b3a67cd2b8b., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731736478770 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731736288173.54bd8d95e5df76bdd2472b3a67cd2b8b. at 1731736478770Flushing 54bd8d95e5df76bdd2472b3a67cd2b8b/colfamily11: creating writer at 1731736478770Flushing 54bd8d95e5df76bdd2472b3a67cd2b8b/colfamily11: appending metadata at 1731736478799 (+29 ms)Flushing 54bd8d95e5df76bdd2472b3a67cd2b8b/colfamily11: closing flushed file at 1731736478799Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e479de8: reopening flushed file at 1731736478857 (+58 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 54bd8d95e5df76bdd2472b3a67cd2b8b in 111ms, sequenceid=5, compaction requested=false at 1731736478880 (+23 ms)Writing region close event to WAL at 1731736478893 (+13 ms)Running coprocessor post-close hooks at 1731736478900 (+7 ms)Closed at 1731736478900 2024-11-16T05:54:38,901 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731736288173.54bd8d95e5df76bdd2472b3a67cd2b8b. 2024-11-16T05:54:38,901 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ec037e054803eda72bad2d81c6143b00, disabling compactions & flushes 2024-11-16T05:54:38,901 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00. 2024-11-16T05:54:38,901 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00. 2024-11-16T05:54:38,901 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00. after waiting 0 ms 2024-11-16T05:54:38,901 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00. 2024-11-16T05:54:38,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741890_1066 (size=39263) 2024-11-16T05:54:38,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741890_1066 (size=39263) 2024-11-16T05:54:38,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741890_1066 (size=39263) 2024-11-16T05:54:38,919 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9df9b8a2b43462b728e2352478a97347/.tmp/colfamily11/3376644a07d547999f275c36b0e6cf73 2024-11-16T05:54:38,924 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ec037e054803eda72bad2d81c6143b00/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-16T05:54:38,927 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00. 2024-11-16T05:54:38,927 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ec037e054803eda72bad2d81c6143b00: Waiting for close lock at 1731736478901Running coprocessor pre-close hooks at 1731736478901Disabling compacts and flushes for region at 1731736478901Disabling writes for close at 1731736478901Writing region close event to WAL at 1731736478912 (+11 ms)Running coprocessor post-close hooks at 1731736478927 (+15 ms)Closed at 1731736478927 2024-11-16T05:54:38,927 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731736288173.ec037e054803eda72bad2d81c6143b00. 2024-11-16T05:54:38,927 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 07f1b4025bc2e2627b7dc3f7c79afd8e, disabling compactions & flushes 2024-11-16T05:54:38,927 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731736288173.07f1b4025bc2e2627b7dc3f7c79afd8e. 2024-11-16T05:54:38,927 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731736288173.07f1b4025bc2e2627b7dc3f7c79afd8e. 2024-11-16T05:54:38,928 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731736288173.07f1b4025bc2e2627b7dc3f7c79afd8e. after waiting 0 ms 2024-11-16T05:54:38,928 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731736288173.07f1b4025bc2e2627b7dc3f7c79afd8e. 2024-11-16T05:54:38,928 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 07f1b4025bc2e2627b7dc3f7c79afd8e 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T05:54:38,930 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9df9b8a2b43462b728e2352478a97347/.tmp/colfamily11/3376644a07d547999f275c36b0e6cf73 as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9df9b8a2b43462b728e2352478a97347/colfamily11/3376644a07d547999f275c36b0e6cf73 2024-11-16T05:54:38,941 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9df9b8a2b43462b728e2352478a97347/colfamily11/3376644a07d547999f275c36b0e6cf73, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T05:54:38,943 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 9df9b8a2b43462b728e2352478a97347 in 83ms, sequenceid=5, compaction requested=false 2024-11-16T05:54:38,961 DEBUG [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer(1351): Waiting on 0166203610c1853876df2910d136e997, 3e30199c6045096f8f565c93af9b6088, 63baab6432268e686bc2f01a98a6c75f, 883c672d1e7de6e08e15c0c42278e66d, 9df9b8a2b43462b728e2352478a97347, ac83f7461cc15e741d0440c6ee2781db, bd79de8195c0452a334d7da789e8f23b, ca14297d921e6ca3fee0822ad2bb732e, de75e055385b7b1b204f0aa9c1924d0e, eb73dbcec883aba378443bd58f1eb724 2024-11-16T05:54:38,961 DEBUG [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(1351): Waiting on 07f1b4025bc2e2627b7dc3f7c79afd8e, 10a3e438f5882d261cd799e4d6b75095, 2beaf8caee4e552158f6bb091a0437bb, 408fc7812ce9cee6ea99428f155ce5a8, 42e678de61063128349f238929e3d296, 53faee5a89035a5faf9a235d93b486c9, afd4d1a884eb847b35b730a1b7a90e0d, df4b7c945ebb852a560a10e20c40c386, ff61a3918e3c512081ba7b9c1e1a4948 2024-11-16T05:54:38,962 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/07f1b4025bc2e2627b7dc3f7c79afd8e/.tmp/colfamily11/cd5d6049bae04fb6b9af18ca9352df7b is 52, key is ooo/colfamily11:colfamily11/1731736292078/Put/seqid=0 2024-11-16T05:54:38,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741891_1067 (size=39263) 2024-11-16T05:54:38,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741891_1067 (size=39263) 2024-11-16T05:54:38,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741891_1067 (size=39263) 2024-11-16T05:54:38,988 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/07f1b4025bc2e2627b7dc3f7c79afd8e/.tmp/colfamily11/cd5d6049bae04fb6b9af18ca9352df7b 2024-11-16T05:54:38,990 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T05:54:38,992 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9df9b8a2b43462b728e2352478a97347/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T05:54:38,993 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731736288173.9df9b8a2b43462b728e2352478a97347. 2024-11-16T05:54:38,993 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9df9b8a2b43462b728e2352478a97347: Waiting for close lock at 1731736478860Running coprocessor pre-close hooks at 1731736478860Disabling compacts and flushes for region at 1731736478860Disabling writes for close at 1731736478860Obtaining lock to block concurrent updates at 1731736478860Preparing flush snapshotting stores in 9df9b8a2b43462b728e2352478a97347 at 1731736478860Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731736288173.9df9b8a2b43462b728e2352478a97347., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731736478861 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731736288173.9df9b8a2b43462b728e2352478a97347. at 1731736478862 (+1 ms)Flushing 9df9b8a2b43462b728e2352478a97347/colfamily11: creating writer at 1731736478862Flushing 9df9b8a2b43462b728e2352478a97347/colfamily11: appending metadata at 1731736478888 (+26 ms)Flushing 9df9b8a2b43462b728e2352478a97347/colfamily11: closing flushed file at 1731736478888Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7856d3a7: reopening flushed file at 1731736478929 (+41 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 9df9b8a2b43462b728e2352478a97347 in 83ms, sequenceid=5, compaction requested=false at 1731736478943 (+14 ms)Writing region close event to WAL at 1731736478967 (+24 ms)Running coprocessor post-close hooks at 1731736478993 (+26 ms)Closed at 1731736478993 2024-11-16T05:54:38,993 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731736288173.9df9b8a2b43462b728e2352478a97347. 2024-11-16T05:54:38,993 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 63baab6432268e686bc2f01a98a6c75f, disabling compactions & flushes 2024-11-16T05:54:38,993 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731736288173.63baab6432268e686bc2f01a98a6c75f. 2024-11-16T05:54:38,994 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731736288173.63baab6432268e686bc2f01a98a6c75f. 2024-11-16T05:54:38,994 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731736288173.63baab6432268e686bc2f01a98a6c75f. after waiting 0 ms 2024-11-16T05:54:38,994 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731736288173.63baab6432268e686bc2f01a98a6c75f. 2024-11-16T05:54:38,994 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 63baab6432268e686bc2f01a98a6c75f 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T05:54:39,006 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/07f1b4025bc2e2627b7dc3f7c79afd8e/.tmp/colfamily11/cd5d6049bae04fb6b9af18ca9352df7b as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/07f1b4025bc2e2627b7dc3f7c79afd8e/colfamily11/cd5d6049bae04fb6b9af18ca9352df7b 2024-11-16T05:54:39,015 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/07f1b4025bc2e2627b7dc3f7c79afd8e/colfamily11/cd5d6049bae04fb6b9af18ca9352df7b, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T05:54:39,016 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 07f1b4025bc2e2627b7dc3f7c79afd8e in 88ms, sequenceid=5, compaction requested=false 2024-11-16T05:54:39,021 INFO [regionserver/3456ee6a3164:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-16T05:54:39,021 INFO [regionserver/3456ee6a3164:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-16T05:54:39,024 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/63baab6432268e686bc2f01a98a6c75f/.tmp/colfamily11/0be73bb63b2d43749dcbde99b4e7d9fa is 52, key is kkk/colfamily11:colfamily11/1731736291981/Put/seqid=0 2024-11-16T05:54:39,030 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/07f1b4025bc2e2627b7dc3f7c79afd8e/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T05:54:39,030 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731736288173.07f1b4025bc2e2627b7dc3f7c79afd8e. 2024-11-16T05:54:39,030 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 07f1b4025bc2e2627b7dc3f7c79afd8e: Waiting for close lock at 1731736478927Running coprocessor pre-close hooks at 1731736478927Disabling compacts and flushes for region at 1731736478927Disabling writes for close at 1731736478928 (+1 ms)Obtaining lock to block concurrent updates at 1731736478928Preparing flush snapshotting stores in 07f1b4025bc2e2627b7dc3f7c79afd8e at 1731736478928Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731736288173.07f1b4025bc2e2627b7dc3f7c79afd8e., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731736478928Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731736288173.07f1b4025bc2e2627b7dc3f7c79afd8e. at 1731736478932 (+4 ms)Flushing 07f1b4025bc2e2627b7dc3f7c79afd8e/colfamily11: creating writer at 1731736478932Flushing 07f1b4025bc2e2627b7dc3f7c79afd8e/colfamily11: appending metadata at 1731736478962 (+30 ms)Flushing 07f1b4025bc2e2627b7dc3f7c79afd8e/colfamily11: closing flushed file at 1731736478962Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5626ef88: reopening flushed file at 1731736479002 (+40 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 07f1b4025bc2e2627b7dc3f7c79afd8e in 88ms, sequenceid=5, compaction requested=false at 1731736479016 (+14 ms)Writing region close event to WAL at 1731736479022 (+6 ms)Running coprocessor post-close hooks at 1731736479030 (+8 ms)Closed at 1731736479030 2024-11-16T05:54:39,031 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731736288173.07f1b4025bc2e2627b7dc3f7c79afd8e. 2024-11-16T05:54:39,031 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing df4b7c945ebb852a560a10e20c40c386, disabling compactions & flushes 2024-11-16T05:54:39,031 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731736288173.df4b7c945ebb852a560a10e20c40c386. 2024-11-16T05:54:39,031 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731736288173.df4b7c945ebb852a560a10e20c40c386. 2024-11-16T05:54:39,031 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731736288173.df4b7c945ebb852a560a10e20c40c386. after waiting 0 ms 2024-11-16T05:54:39,031 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731736288173.df4b7c945ebb852a560a10e20c40c386. 2024-11-16T05:54:39,031 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing df4b7c945ebb852a560a10e20c40c386 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T05:54:39,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741892_1068 (size=39263) 2024-11-16T05:54:39,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741892_1068 (size=39263) 2024-11-16T05:54:39,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741892_1068 (size=39263) 2024-11-16T05:54:39,049 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/63baab6432268e686bc2f01a98a6c75f/.tmp/colfamily11/0be73bb63b2d43749dcbde99b4e7d9fa 2024-11-16T05:54:39,061 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/63baab6432268e686bc2f01a98a6c75f/.tmp/colfamily11/0be73bb63b2d43749dcbde99b4e7d9fa as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/63baab6432268e686bc2f01a98a6c75f/colfamily11/0be73bb63b2d43749dcbde99b4e7d9fa 2024-11-16T05:54:39,061 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/df4b7c945ebb852a560a10e20c40c386/.tmp/colfamily11/632fd027d2a546afa1b52db60797a55d is 52, key is rrr/colfamily11:colfamily11/1731736292118/Put/seqid=0 2024-11-16T05:54:39,068 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/63baab6432268e686bc2f01a98a6c75f/colfamily11/0be73bb63b2d43749dcbde99b4e7d9fa, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T05:54:39,069 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 63baab6432268e686bc2f01a98a6c75f in 75ms, sequenceid=5, compaction requested=false 2024-11-16T05:54:39,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741893_1069 (size=39263) 2024-11-16T05:54:39,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741893_1069 (size=39263) 2024-11-16T05:54:39,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741893_1069 (size=39263) 2024-11-16T05:54:39,072 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/df4b7c945ebb852a560a10e20c40c386/.tmp/colfamily11/632fd027d2a546afa1b52db60797a55d 2024-11-16T05:54:39,076 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/63baab6432268e686bc2f01a98a6c75f/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T05:54:39,076 INFO [regionserver/3456ee6a3164:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-16T05:54:39,077 INFO [regionserver/3456ee6a3164:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-16T05:54:39,077 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731736288173.63baab6432268e686bc2f01a98a6c75f. 2024-11-16T05:54:39,077 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 63baab6432268e686bc2f01a98a6c75f: Waiting for close lock at 1731736478993Running coprocessor pre-close hooks at 1731736478993Disabling compacts and flushes for region at 1731736478993Disabling writes for close at 1731736478994 (+1 ms)Obtaining lock to block concurrent updates at 1731736478994Preparing flush snapshotting stores in 63baab6432268e686bc2f01a98a6c75f at 1731736478994Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731736288173.63baab6432268e686bc2f01a98a6c75f., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731736478995 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731736288173.63baab6432268e686bc2f01a98a6c75f. at 1731736478999 (+4 ms)Flushing 63baab6432268e686bc2f01a98a6c75f/colfamily11: creating writer at 1731736478999Flushing 63baab6432268e686bc2f01a98a6c75f/colfamily11: appending metadata at 1731736479023 (+24 ms)Flushing 63baab6432268e686bc2f01a98a6c75f/colfamily11: closing flushed file at 1731736479023Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c4cac0: reopening flushed file at 1731736479060 (+37 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 63baab6432268e686bc2f01a98a6c75f in 75ms, sequenceid=5, compaction requested=false at 1731736479069 (+9 ms)Writing region close event to WAL at 1731736479070 (+1 ms)Running coprocessor post-close hooks at 1731736479077 (+7 ms)Closed at 1731736479077 2024-11-16T05:54:39,077 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731736288173.63baab6432268e686bc2f01a98a6c75f. 2024-11-16T05:54:39,077 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 883c672d1e7de6e08e15c0c42278e66d, disabling compactions & flushes 2024-11-16T05:54:39,077 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d. 2024-11-16T05:54:39,077 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d. 2024-11-16T05:54:39,077 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d. after waiting 0 ms 2024-11-16T05:54:39,077 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d. 2024-11-16T05:54:39,080 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/df4b7c945ebb852a560a10e20c40c386/.tmp/colfamily11/632fd027d2a546afa1b52db60797a55d as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/df4b7c945ebb852a560a10e20c40c386/colfamily11/632fd027d2a546afa1b52db60797a55d 2024-11-16T05:54:39,082 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/883c672d1e7de6e08e15c0c42278e66d/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-16T05:54:39,083 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d. 2024-11-16T05:54:39,083 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 883c672d1e7de6e08e15c0c42278e66d: Waiting for close lock at 1731736479077Running coprocessor pre-close hooks at 1731736479077Disabling compacts and flushes for region at 1731736479077Disabling writes for close at 1731736479077Writing region close event to WAL at 1731736479078 (+1 ms)Running coprocessor post-close hooks at 1731736479082 (+4 ms)Closed at 1731736479082 2024-11-16T05:54:39,083 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731736288173.883c672d1e7de6e08e15c0c42278e66d. 2024-11-16T05:54:39,083 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ac83f7461cc15e741d0440c6ee2781db, disabling compactions & flushes 2024-11-16T05:54:39,083 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db. 2024-11-16T05:54:39,083 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db. 2024-11-16T05:54:39,083 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db. after waiting 0 ms 2024-11-16T05:54:39,083 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db. 2024-11-16T05:54:39,087 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/df4b7c945ebb852a560a10e20c40c386/colfamily11/632fd027d2a546afa1b52db60797a55d, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T05:54:39,088 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac83f7461cc15e741d0440c6ee2781db/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-16T05:54:39,089 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for df4b7c945ebb852a560a10e20c40c386 in 57ms, sequenceid=5, compaction requested=false 2024-11-16T05:54:39,089 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db. 2024-11-16T05:54:39,089 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ac83f7461cc15e741d0440c6ee2781db: Waiting for close lock at 1731736479083Running coprocessor pre-close hooks at 1731736479083Disabling compacts and flushes for region at 1731736479083Disabling writes for close at 1731736479083Writing region close event to WAL at 1731736479084 (+1 ms)Running coprocessor post-close hooks at 1731736479089 (+5 ms)Closed at 1731736479089 2024-11-16T05:54:39,089 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731736288173.ac83f7461cc15e741d0440c6ee2781db. 2024-11-16T05:54:39,090 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ca14297d921e6ca3fee0822ad2bb732e, disabling compactions & flushes 2024-11-16T05:54:39,090 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731736288173.ca14297d921e6ca3fee0822ad2bb732e. 2024-11-16T05:54:39,090 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731736288173.ca14297d921e6ca3fee0822ad2bb732e. 2024-11-16T05:54:39,090 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731736288173.ca14297d921e6ca3fee0822ad2bb732e. after waiting 0 ms 2024-11-16T05:54:39,090 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731736288173.ca14297d921e6ca3fee0822ad2bb732e. 2024-11-16T05:54:39,090 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing ca14297d921e6ca3fee0822ad2bb732e 3/3 column families, dataSize=48 B heapSize=888 B 2024-11-16T05:54:39,093 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/df4b7c945ebb852a560a10e20c40c386/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T05:54:39,094 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731736288173.df4b7c945ebb852a560a10e20c40c386. 2024-11-16T05:54:39,094 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for df4b7c945ebb852a560a10e20c40c386: Waiting for close lock at 1731736479031Running coprocessor pre-close hooks at 1731736479031Disabling compacts and flushes for region at 1731736479031Disabling writes for close at 1731736479031Obtaining lock to block concurrent updates at 1731736479031Preparing flush snapshotting stores in df4b7c945ebb852a560a10e20c40c386 at 1731736479031Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731736288173.df4b7c945ebb852a560a10e20c40c386., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731736479031Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731736288173.df4b7c945ebb852a560a10e20c40c386. at 1731736479032 (+1 ms)Flushing df4b7c945ebb852a560a10e20c40c386/colfamily11: creating writer at 1731736479032Flushing df4b7c945ebb852a560a10e20c40c386/colfamily11: appending metadata at 1731736479061 (+29 ms)Flushing df4b7c945ebb852a560a10e20c40c386/colfamily11: closing flushed file at 1731736479061Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70fb1f9b: reopening flushed file at 1731736479079 (+18 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for df4b7c945ebb852a560a10e20c40c386 in 57ms, sequenceid=5, compaction requested=false at 1731736479089 (+10 ms)Writing region close event to WAL at 1731736479089Running coprocessor post-close hooks at 1731736479094 (+5 ms)Closed at 1731736479094 2024-11-16T05:54:39,095 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731736288173.df4b7c945ebb852a560a10e20c40c386. 2024-11-16T05:54:39,095 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 53faee5a89035a5faf9a235d93b486c9, disabling compactions & flushes 2024-11-16T05:54:39,095 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731736288173.53faee5a89035a5faf9a235d93b486c9. 2024-11-16T05:54:39,095 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731736288173.53faee5a89035a5faf9a235d93b486c9. 2024-11-16T05:54:39,095 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731736288173.53faee5a89035a5faf9a235d93b486c9. after waiting 0 ms 2024-11-16T05:54:39,095 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731736288173.53faee5a89035a5faf9a235d93b486c9. 2024-11-16T05:54:39,095 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 53faee5a89035a5faf9a235d93b486c9 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T05:54:39,107 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ca14297d921e6ca3fee0822ad2bb732e/.tmp/colfamily11/7b497973c5b04e839ceee3a1e6a32f90 is 52, key is zzz/colfamily11:colfamily11/1731736292144/Put/seqid=0 2024-11-16T05:54:39,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741894_1070 (size=4857) 2024-11-16T05:54:39,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741894_1070 (size=4857) 2024-11-16T05:54:39,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741894_1070 (size=4857) 2024-11-16T05:54:39,121 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ca14297d921e6ca3fee0822ad2bb732e/.tmp/colfamily11/7b497973c5b04e839ceee3a1e6a32f90 2024-11-16T05:54:39,123 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/53faee5a89035a5faf9a235d93b486c9/.tmp/colfamily11/46a33adb9d064d88bcb3316134d7d0f9 is 52, key is eee/colfamily11:colfamily11/1731736291890/Put/seqid=0 2024-11-16T05:54:39,128 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ca14297d921e6ca3fee0822ad2bb732e/.tmp/colfamily11/7b497973c5b04e839ceee3a1e6a32f90 as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ca14297d921e6ca3fee0822ad2bb732e/colfamily11/7b497973c5b04e839ceee3a1e6a32f90 2024-11-16T05:54:39,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741895_1071 (size=39263) 2024-11-16T05:54:39,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741895_1071 (size=39263) 2024-11-16T05:54:39,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741895_1071 (size=39263) 2024-11-16T05:54:39,133 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/53faee5a89035a5faf9a235d93b486c9/.tmp/colfamily11/46a33adb9d064d88bcb3316134d7d0f9 2024-11-16T05:54:39,136 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ca14297d921e6ca3fee0822ad2bb732e/colfamily11/7b497973c5b04e839ceee3a1e6a32f90, entries=1, sequenceid=5, filesize=4.7 K 2024-11-16T05:54:39,137 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~48 B/48, heapSize ~360 B/360, currentSize=0 B/0 for ca14297d921e6ca3fee0822ad2bb732e in 47ms, sequenceid=5, compaction requested=false 2024-11-16T05:54:39,141 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/53faee5a89035a5faf9a235d93b486c9/.tmp/colfamily11/46a33adb9d064d88bcb3316134d7d0f9 as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/53faee5a89035a5faf9a235d93b486c9/colfamily11/46a33adb9d064d88bcb3316134d7d0f9 2024-11-16T05:54:39,142 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ca14297d921e6ca3fee0822ad2bb732e/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T05:54:39,142 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731736288173.ca14297d921e6ca3fee0822ad2bb732e. 2024-11-16T05:54:39,143 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ca14297d921e6ca3fee0822ad2bb732e: Waiting for close lock at 1731736479089Running coprocessor pre-close hooks at 1731736479089Disabling compacts and flushes for region at 1731736479090 (+1 ms)Disabling writes for close at 1731736479090Obtaining lock to block concurrent updates at 1731736479090Preparing flush snapshotting stores in ca14297d921e6ca3fee0822ad2bb732e at 1731736479090Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731736288173.ca14297d921e6ca3fee0822ad2bb732e., syncing WAL and waiting on mvcc, flushsize=dataSize=48, getHeapSize=840, getOffHeapSize=0, getCellsCount=1 at 1731736479090Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731736288173.ca14297d921e6ca3fee0822ad2bb732e. at 1731736479091 (+1 ms)Flushing ca14297d921e6ca3fee0822ad2bb732e/colfamily11: creating writer at 1731736479091Flushing ca14297d921e6ca3fee0822ad2bb732e/colfamily11: appending metadata at 1731736479107 (+16 ms)Flushing ca14297d921e6ca3fee0822ad2bb732e/colfamily11: closing flushed file at 1731736479107Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3241cc83: reopening flushed file at 1731736479127 (+20 ms)Finished flush of dataSize ~48 B/48, heapSize ~360 B/360, currentSize=0 B/0 for ca14297d921e6ca3fee0822ad2bb732e in 47ms, sequenceid=5, compaction requested=false at 1731736479137 (+10 ms)Writing region close event to WAL at 1731736479138 (+1 ms)Running coprocessor post-close hooks at 1731736479142 (+4 ms)Closed at 1731736479142 2024-11-16T05:54:39,143 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731736288173.ca14297d921e6ca3fee0822ad2bb732e. 2024-11-16T05:54:39,143 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0166203610c1853876df2910d136e997, disabling compactions & flushes 2024-11-16T05:54:39,143 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731736288173.0166203610c1853876df2910d136e997. 2024-11-16T05:54:39,143 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731736288173.0166203610c1853876df2910d136e997. 2024-11-16T05:54:39,143 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731736288173.0166203610c1853876df2910d136e997. after waiting 0 ms 2024-11-16T05:54:39,143 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731736288173.0166203610c1853876df2910d136e997. 2024-11-16T05:54:39,143 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 0166203610c1853876df2910d136e997 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T05:54:39,148 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/53faee5a89035a5faf9a235d93b486c9/colfamily11/46a33adb9d064d88bcb3316134d7d0f9, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T05:54:39,149 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 53faee5a89035a5faf9a235d93b486c9 in 54ms, sequenceid=5, compaction requested=false 2024-11-16T05:54:39,154 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/53faee5a89035a5faf9a235d93b486c9/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T05:54:39,155 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731736288173.53faee5a89035a5faf9a235d93b486c9. 2024-11-16T05:54:39,155 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 53faee5a89035a5faf9a235d93b486c9: Waiting for close lock at 1731736479095Running coprocessor pre-close hooks at 1731736479095Disabling compacts and flushes for region at 1731736479095Disabling writes for close at 1731736479095Obtaining lock to block concurrent updates at 1731736479095Preparing flush snapshotting stores in 53faee5a89035a5faf9a235d93b486c9 at 1731736479095Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731736288173.53faee5a89035a5faf9a235d93b486c9., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731736479095Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731736288173.53faee5a89035a5faf9a235d93b486c9. at 1731736479096 (+1 ms)Flushing 53faee5a89035a5faf9a235d93b486c9/colfamily11: creating writer at 1731736479096Flushing 53faee5a89035a5faf9a235d93b486c9/colfamily11: appending metadata at 1731736479123 (+27 ms)Flushing 53faee5a89035a5faf9a235d93b486c9/colfamily11: closing flushed file at 1731736479123Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@239a6e03: reopening flushed file at 1731736479140 (+17 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 53faee5a89035a5faf9a235d93b486c9 in 54ms, sequenceid=5, compaction requested=false at 1731736479149 (+9 ms)Writing region close event to WAL at 1731736479150 (+1 ms)Running coprocessor post-close hooks at 1731736479155 (+5 ms)Closed at 1731736479155 2024-11-16T05:54:39,155 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731736288173.53faee5a89035a5faf9a235d93b486c9. 2024-11-16T05:54:39,155 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 408fc7812ce9cee6ea99428f155ce5a8, disabling compactions & flushes 2024-11-16T05:54:39,156 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731736288173.408fc7812ce9cee6ea99428f155ce5a8. 2024-11-16T05:54:39,156 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731736288173.408fc7812ce9cee6ea99428f155ce5a8. 2024-11-16T05:54:39,156 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731736288173.408fc7812ce9cee6ea99428f155ce5a8. after waiting 0 ms 2024-11-16T05:54:39,156 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731736288173.408fc7812ce9cee6ea99428f155ce5a8. 2024-11-16T05:54:39,156 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 408fc7812ce9cee6ea99428f155ce5a8 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T05:54:39,161 DEBUG [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer(1351): Waiting on 0166203610c1853876df2910d136e997, 3e30199c6045096f8f565c93af9b6088, bd79de8195c0452a334d7da789e8f23b, de75e055385b7b1b204f0aa9c1924d0e, eb73dbcec883aba378443bd58f1eb724 2024-11-16T05:54:39,161 DEBUG [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(1351): Waiting on 10a3e438f5882d261cd799e4d6b75095, 2beaf8caee4e552158f6bb091a0437bb, 408fc7812ce9cee6ea99428f155ce5a8, 42e678de61063128349f238929e3d296, afd4d1a884eb847b35b730a1b7a90e0d, ff61a3918e3c512081ba7b9c1e1a4948 2024-11-16T05:54:39,162 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0166203610c1853876df2910d136e997/.tmp/colfamily11/7a93cb70899440cab30877cb625cb9ff is 52, key is qqq/colfamily11:colfamily11/1731736292057/Put/seqid=0 2024-11-16T05:54:39,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741896_1072 (size=39263) 2024-11-16T05:54:39,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741896_1072 (size=39263) 2024-11-16T05:54:39,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741896_1072 (size=39263) 2024-11-16T05:54:39,181 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0166203610c1853876df2910d136e997/.tmp/colfamily11/7a93cb70899440cab30877cb625cb9ff 2024-11-16T05:54:39,182 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/408fc7812ce9cee6ea99428f155ce5a8/.tmp/colfamily11/0713c51d728a4a1cbee738a5f64101a3 is 52, key is lll/colfamily11:colfamily11/1731736292040/Put/seqid=0 2024-11-16T05:54:39,190 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0166203610c1853876df2910d136e997/.tmp/colfamily11/7a93cb70899440cab30877cb625cb9ff as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0166203610c1853876df2910d136e997/colfamily11/7a93cb70899440cab30877cb625cb9ff 2024-11-16T05:54:39,196 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0166203610c1853876df2910d136e997/colfamily11/7a93cb70899440cab30877cb625cb9ff, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T05:54:39,198 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 0166203610c1853876df2910d136e997 in 54ms, sequenceid=5, compaction requested=false 2024-11-16T05:54:39,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741897_1073 (size=39263) 2024-11-16T05:54:39,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741897_1073 (size=39263) 2024-11-16T05:54:39,202 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/408fc7812ce9cee6ea99428f155ce5a8/.tmp/colfamily11/0713c51d728a4a1cbee738a5f64101a3 2024-11-16T05:54:39,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741897_1073 (size=39263) 2024-11-16T05:54:39,205 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/0166203610c1853876df2910d136e997/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T05:54:39,205 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731736288173.0166203610c1853876df2910d136e997. 2024-11-16T05:54:39,206 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0166203610c1853876df2910d136e997: Waiting for close lock at 1731736479143Running coprocessor pre-close hooks at 1731736479143Disabling compacts and flushes for region at 1731736479143Disabling writes for close at 1731736479143Obtaining lock to block concurrent updates at 1731736479143Preparing flush snapshotting stores in 0166203610c1853876df2910d136e997 at 1731736479143Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731736288173.0166203610c1853876df2910d136e997., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731736479143Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731736288173.0166203610c1853876df2910d136e997. at 1731736479144 (+1 ms)Flushing 0166203610c1853876df2910d136e997/colfamily11: creating writer at 1731736479144Flushing 0166203610c1853876df2910d136e997/colfamily11: appending metadata at 1731736479161 (+17 ms)Flushing 0166203610c1853876df2910d136e997/colfamily11: closing flushed file at 1731736479162 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32666bec: reopening flushed file at 1731736479189 (+27 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 0166203610c1853876df2910d136e997 in 54ms, sequenceid=5, compaction requested=false at 1731736479198 (+9 ms)Writing region close event to WAL at 1731736479201 (+3 ms)Running coprocessor post-close hooks at 1731736479205 (+4 ms)Closed at 1731736479205 2024-11-16T05:54:39,206 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731736288173.0166203610c1853876df2910d136e997. 2024-11-16T05:54:39,206 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3e30199c6045096f8f565c93af9b6088, disabling compactions & flushes 2024-11-16T05:54:39,206 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731736288173.3e30199c6045096f8f565c93af9b6088. 2024-11-16T05:54:39,206 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731736288173.3e30199c6045096f8f565c93af9b6088. 2024-11-16T05:54:39,206 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731736288173.3e30199c6045096f8f565c93af9b6088. after waiting 0 ms 2024-11-16T05:54:39,206 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731736288173.3e30199c6045096f8f565c93af9b6088. 2024-11-16T05:54:39,206 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 3e30199c6045096f8f565c93af9b6088 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T05:54:39,209 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/408fc7812ce9cee6ea99428f155ce5a8/.tmp/colfamily11/0713c51d728a4a1cbee738a5f64101a3 as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/408fc7812ce9cee6ea99428f155ce5a8/colfamily11/0713c51d728a4a1cbee738a5f64101a3 2024-11-16T05:54:39,216 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/408fc7812ce9cee6ea99428f155ce5a8/colfamily11/0713c51d728a4a1cbee738a5f64101a3, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T05:54:39,217 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 408fc7812ce9cee6ea99428f155ce5a8 in 61ms, sequenceid=5, compaction requested=false 2024-11-16T05:54:39,222 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/408fc7812ce9cee6ea99428f155ce5a8/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T05:54:39,223 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731736288173.408fc7812ce9cee6ea99428f155ce5a8. 2024-11-16T05:54:39,223 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 408fc7812ce9cee6ea99428f155ce5a8: Waiting for close lock at 1731736479155Running coprocessor pre-close hooks at 1731736479155Disabling compacts and flushes for region at 1731736479155Disabling writes for close at 1731736479156 (+1 ms)Obtaining lock to block concurrent updates at 1731736479156Preparing flush snapshotting stores in 408fc7812ce9cee6ea99428f155ce5a8 at 1731736479156Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731736288173.408fc7812ce9cee6ea99428f155ce5a8., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731736479156Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731736288173.408fc7812ce9cee6ea99428f155ce5a8. at 1731736479161 (+5 ms)Flushing 408fc7812ce9cee6ea99428f155ce5a8/colfamily11: creating writer at 1731736479161Flushing 408fc7812ce9cee6ea99428f155ce5a8/colfamily11: appending metadata at 1731736479181 (+20 ms)Flushing 408fc7812ce9cee6ea99428f155ce5a8/colfamily11: closing flushed file at 1731736479182 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@403dc9a9: reopening flushed file at 1731736479208 (+26 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 408fc7812ce9cee6ea99428f155ce5a8 in 61ms, sequenceid=5, compaction requested=false at 1731736479217 (+9 ms)Writing region close event to WAL at 1731736479218 (+1 ms)Running coprocessor post-close hooks at 1731736479223 (+5 ms)Closed at 1731736479223 2024-11-16T05:54:39,223 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731736288173.408fc7812ce9cee6ea99428f155ce5a8. 2024-11-16T05:54:39,223 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ff61a3918e3c512081ba7b9c1e1a4948, disabling compactions & flushes 2024-11-16T05:54:39,223 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731736288173.ff61a3918e3c512081ba7b9c1e1a4948. 2024-11-16T05:54:39,223 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731736288173.ff61a3918e3c512081ba7b9c1e1a4948. 2024-11-16T05:54:39,223 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731736288173.ff61a3918e3c512081ba7b9c1e1a4948. after waiting 0 ms 2024-11-16T05:54:39,224 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731736288173.ff61a3918e3c512081ba7b9c1e1a4948. 2024-11-16T05:54:39,224 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing ff61a3918e3c512081ba7b9c1e1a4948 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T05:54:39,230 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3e30199c6045096f8f565c93af9b6088/.tmp/colfamily11/fd19ce8ea6fc4833b80696a757eabd3b is 52, key is fff/colfamily11:colfamily11/1731736291950/Put/seqid=0 2024-11-16T05:54:39,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741898_1074 (size=39263) 2024-11-16T05:54:39,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741898_1074 (size=39263) 2024-11-16T05:54:39,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741898_1074 (size=39263) 2024-11-16T05:54:39,239 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3e30199c6045096f8f565c93af9b6088/.tmp/colfamily11/fd19ce8ea6fc4833b80696a757eabd3b 2024-11-16T05:54:39,247 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3e30199c6045096f8f565c93af9b6088/.tmp/colfamily11/fd19ce8ea6fc4833b80696a757eabd3b as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3e30199c6045096f8f565c93af9b6088/colfamily11/fd19ce8ea6fc4833b80696a757eabd3b 2024-11-16T05:54:39,248 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ff61a3918e3c512081ba7b9c1e1a4948/.tmp/colfamily11/f24cf883c2fe4bfb92ad9e5a4efb17a1 is 52, key is hhh/colfamily11:colfamily11/1731736291960/Put/seqid=0 2024-11-16T05:54:39,255 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3e30199c6045096f8f565c93af9b6088/colfamily11/fd19ce8ea6fc4833b80696a757eabd3b, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T05:54:39,256 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 3e30199c6045096f8f565c93af9b6088 in 49ms, sequenceid=5, compaction requested=false 2024-11-16T05:54:39,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741899_1075 (size=39263) 2024-11-16T05:54:39,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741899_1075 (size=39263) 2024-11-16T05:54:39,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741899_1075 (size=39263) 2024-11-16T05:54:39,262 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ff61a3918e3c512081ba7b9c1e1a4948/.tmp/colfamily11/f24cf883c2fe4bfb92ad9e5a4efb17a1 2024-11-16T05:54:39,265 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/3e30199c6045096f8f565c93af9b6088/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T05:54:39,266 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731736288173.3e30199c6045096f8f565c93af9b6088. 2024-11-16T05:54:39,266 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3e30199c6045096f8f565c93af9b6088: Waiting for close lock at 1731736479206Running coprocessor pre-close hooks at 1731736479206Disabling compacts and flushes for region at 1731736479206Disabling writes for close at 1731736479206Obtaining lock to block concurrent updates at 1731736479206Preparing flush snapshotting stores in 3e30199c6045096f8f565c93af9b6088 at 1731736479206Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731736288173.3e30199c6045096f8f565c93af9b6088., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731736479207 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731736288173.3e30199c6045096f8f565c93af9b6088. at 1731736479208 (+1 ms)Flushing 3e30199c6045096f8f565c93af9b6088/colfamily11: creating writer at 1731736479208Flushing 3e30199c6045096f8f565c93af9b6088/colfamily11: appending metadata at 1731736479230 (+22 ms)Flushing 3e30199c6045096f8f565c93af9b6088/colfamily11: closing flushed file at 1731736479230Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23aef80d: reopening flushed file at 1731736479246 (+16 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 3e30199c6045096f8f565c93af9b6088 in 49ms, sequenceid=5, compaction requested=false at 1731736479256 (+10 ms)Writing region close event to WAL at 1731736479260 (+4 ms)Running coprocessor post-close hooks at 1731736479266 (+6 ms)Closed at 1731736479266 2024-11-16T05:54:39,266 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731736288173.3e30199c6045096f8f565c93af9b6088. 2024-11-16T05:54:39,266 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eb73dbcec883aba378443bd58f1eb724, disabling compactions & flushes 2024-11-16T05:54:39,266 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731736288173.eb73dbcec883aba378443bd58f1eb724. 2024-11-16T05:54:39,266 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731736288173.eb73dbcec883aba378443bd58f1eb724. 2024-11-16T05:54:39,266 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731736288173.eb73dbcec883aba378443bd58f1eb724. after waiting 0 ms 2024-11-16T05:54:39,266 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731736288173.eb73dbcec883aba378443bd58f1eb724. 2024-11-16T05:54:39,266 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing eb73dbcec883aba378443bd58f1eb724 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T05:54:39,273 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ff61a3918e3c512081ba7b9c1e1a4948/.tmp/colfamily11/f24cf883c2fe4bfb92ad9e5a4efb17a1 as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ff61a3918e3c512081ba7b9c1e1a4948/colfamily11/f24cf883c2fe4bfb92ad9e5a4efb17a1 2024-11-16T05:54:39,296 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/eb73dbcec883aba378443bd58f1eb724/.tmp/colfamily11/dfaf66d8dcc544e18b74b38c7b5e0fb2 is 52, key is www/colfamily11:colfamily11/1731736292137/Put/seqid=0 2024-11-16T05:54:39,302 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ff61a3918e3c512081ba7b9c1e1a4948/colfamily11/f24cf883c2fe4bfb92ad9e5a4efb17a1, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T05:54:39,303 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for ff61a3918e3c512081ba7b9c1e1a4948 in 79ms, sequenceid=5, compaction requested=false 2024-11-16T05:54:39,337 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ff61a3918e3c512081ba7b9c1e1a4948/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T05:54:39,339 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731736288173.ff61a3918e3c512081ba7b9c1e1a4948. 2024-11-16T05:54:39,339 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ff61a3918e3c512081ba7b9c1e1a4948: Waiting for close lock at 1731736479223Running coprocessor pre-close hooks at 1731736479223Disabling compacts and flushes for region at 1731736479223Disabling writes for close at 1731736479223Obtaining lock to block concurrent updates at 1731736479224 (+1 ms)Preparing flush snapshotting stores in ff61a3918e3c512081ba7b9c1e1a4948 at 1731736479224Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731736288173.ff61a3918e3c512081ba7b9c1e1a4948., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731736479224Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731736288173.ff61a3918e3c512081ba7b9c1e1a4948. at 1731736479225 (+1 ms)Flushing ff61a3918e3c512081ba7b9c1e1a4948/colfamily11: creating writer at 1731736479225Flushing ff61a3918e3c512081ba7b9c1e1a4948/colfamily11: appending metadata at 1731736479247 (+22 ms)Flushing ff61a3918e3c512081ba7b9c1e1a4948/colfamily11: closing flushed file at 1731736479247Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f264015: reopening flushed file at 1731736479270 (+23 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for ff61a3918e3c512081ba7b9c1e1a4948 in 79ms, sequenceid=5, compaction requested=false at 1731736479303 (+33 ms)Writing region close event to WAL at 1731736479309 (+6 ms)Running coprocessor post-close hooks at 1731736479339 (+30 ms)Closed at 1731736479339 2024-11-16T05:54:39,339 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731736288173.ff61a3918e3c512081ba7b9c1e1a4948. 2024-11-16T05:54:39,339 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 10a3e438f5882d261cd799e4d6b75095, disabling compactions & flushes 2024-11-16T05:54:39,339 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095. 2024-11-16T05:54:39,339 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095. 2024-11-16T05:54:39,339 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095. after waiting 0 ms 2024-11-16T05:54:39,340 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095. 2024-11-16T05:54:39,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741900_1076 (size=39263) 2024-11-16T05:54:39,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741900_1076 (size=39263) 2024-11-16T05:54:39,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741900_1076 (size=39263) 2024-11-16T05:54:39,342 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/eb73dbcec883aba378443bd58f1eb724/.tmp/colfamily11/dfaf66d8dcc544e18b74b38c7b5e0fb2 2024-11-16T05:54:39,352 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/eb73dbcec883aba378443bd58f1eb724/.tmp/colfamily11/dfaf66d8dcc544e18b74b38c7b5e0fb2 as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/eb73dbcec883aba378443bd58f1eb724/colfamily11/dfaf66d8dcc544e18b74b38c7b5e0fb2 2024-11-16T05:54:39,358 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/10a3e438f5882d261cd799e4d6b75095/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-16T05:54:39,359 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095. 2024-11-16T05:54:39,359 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 10a3e438f5882d261cd799e4d6b75095: Waiting for close lock at 1731736479339Running coprocessor pre-close hooks at 1731736479339Disabling compacts and flushes for region at 1731736479339Disabling writes for close at 1731736479339Writing region close event to WAL at 1731736479345 (+6 ms)Running coprocessor post-close hooks at 1731736479359 (+14 ms)Closed at 1731736479359 2024-11-16T05:54:39,359 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731736288173.10a3e438f5882d261cd799e4d6b75095. 2024-11-16T05:54:39,359 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 42e678de61063128349f238929e3d296, disabling compactions & flushes 2024-11-16T05:54:39,359 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296. 2024-11-16T05:54:39,359 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296. 2024-11-16T05:54:39,360 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296. after waiting 0 ms 2024-11-16T05:54:39,360 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296. 2024-11-16T05:54:39,360 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/eb73dbcec883aba378443bd58f1eb724/colfamily11/dfaf66d8dcc544e18b74b38c7b5e0fb2, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T05:54:39,361 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for eb73dbcec883aba378443bd58f1eb724 in 95ms, sequenceid=5, compaction requested=false 2024-11-16T05:54:39,362 DEBUG [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer(1351): Waiting on bd79de8195c0452a334d7da789e8f23b, de75e055385b7b1b204f0aa9c1924d0e, eb73dbcec883aba378443bd58f1eb724 2024-11-16T05:54:39,362 DEBUG [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(1351): Waiting on 2beaf8caee4e552158f6bb091a0437bb, 42e678de61063128349f238929e3d296, afd4d1a884eb847b35b730a1b7a90e0d 2024-11-16T05:54:39,370 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/42e678de61063128349f238929e3d296/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-16T05:54:39,371 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296. 2024-11-16T05:54:39,371 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 42e678de61063128349f238929e3d296: Waiting for close lock at 1731736479359Running coprocessor pre-close hooks at 1731736479359Disabling compacts and flushes for region at 1731736479359Disabling writes for close at 1731736479360 (+1 ms)Writing region close event to WAL at 1731736479365 (+5 ms)Running coprocessor post-close hooks at 1731736479370 (+5 ms)Closed at 1731736479370 2024-11-16T05:54:39,371 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731736288173.42e678de61063128349f238929e3d296. 2024-11-16T05:54:39,371 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2beaf8caee4e552158f6bb091a0437bb, disabling compactions & flushes 2024-11-16T05:54:39,371 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731736288173.2beaf8caee4e552158f6bb091a0437bb. 2024-11-16T05:54:39,371 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731736288173.2beaf8caee4e552158f6bb091a0437bb. 2024-11-16T05:54:39,371 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731736288173.2beaf8caee4e552158f6bb091a0437bb. after waiting 0 ms 2024-11-16T05:54:39,371 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731736288173.2beaf8caee4e552158f6bb091a0437bb. 2024-11-16T05:54:39,371 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 2beaf8caee4e552158f6bb091a0437bb 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T05:54:39,371 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/eb73dbcec883aba378443bd58f1eb724/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T05:54:39,372 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731736288173.eb73dbcec883aba378443bd58f1eb724. 2024-11-16T05:54:39,372 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eb73dbcec883aba378443bd58f1eb724: Waiting for close lock at 1731736479266Running coprocessor pre-close hooks at 1731736479266Disabling compacts and flushes for region at 1731736479266Disabling writes for close at 1731736479266Obtaining lock to block concurrent updates at 1731736479266Preparing flush snapshotting stores in eb73dbcec883aba378443bd58f1eb724 at 1731736479266Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731736288173.eb73dbcec883aba378443bd58f1eb724., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731736479267 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731736288173.eb73dbcec883aba378443bd58f1eb724. at 1731736479268 (+1 ms)Flushing eb73dbcec883aba378443bd58f1eb724/colfamily11: creating writer at 1731736479268Flushing eb73dbcec883aba378443bd58f1eb724/colfamily11: appending metadata at 1731736479296 (+28 ms)Flushing eb73dbcec883aba378443bd58f1eb724/colfamily11: closing flushed file at 1731736479296Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d8ef619: reopening flushed file at 1731736479351 (+55 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for eb73dbcec883aba378443bd58f1eb724 in 95ms, sequenceid=5, compaction requested=false at 1731736479361 (+10 ms)Writing region close event to WAL at 1731736479367 (+6 ms)Running coprocessor post-close hooks at 1731736479372 (+5 ms)Closed at 1731736479372 2024-11-16T05:54:39,372 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731736288173.eb73dbcec883aba378443bd58f1eb724. 2024-11-16T05:54:39,373 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bd79de8195c0452a334d7da789e8f23b, disabling compactions & flushes 2024-11-16T05:54:39,373 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b. 2024-11-16T05:54:39,373 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b. 2024-11-16T05:54:39,373 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b. after waiting 0 ms 2024-11-16T05:54:39,373 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b. 2024-11-16T05:54:39,390 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bd79de8195c0452a334d7da789e8f23b/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-16T05:54:39,391 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b. 2024-11-16T05:54:39,391 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bd79de8195c0452a334d7da789e8f23b: Waiting for close lock at 1731736479373Running coprocessor pre-close hooks at 1731736479373Disabling compacts and flushes for region at 1731736479373Disabling writes for close at 1731736479373Writing region close event to WAL at 1731736479374 (+1 ms)Running coprocessor post-close hooks at 1731736479391 (+17 ms)Closed at 1731736479391 2024-11-16T05:54:39,391 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731736288173.bd79de8195c0452a334d7da789e8f23b. 2024-11-16T05:54:39,391 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing de75e055385b7b1b204f0aa9c1924d0e, disabling compactions & flushes 2024-11-16T05:54:39,391 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731736288173.de75e055385b7b1b204f0aa9c1924d0e. 2024-11-16T05:54:39,391 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731736288173.de75e055385b7b1b204f0aa9c1924d0e. 2024-11-16T05:54:39,391 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731736288173.de75e055385b7b1b204f0aa9c1924d0e. after waiting 0 ms 2024-11-16T05:54:39,391 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731736288173.de75e055385b7b1b204f0aa9c1924d0e. 2024-11-16T05:54:39,391 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing de75e055385b7b1b204f0aa9c1924d0e 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T05:54:39,402 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2beaf8caee4e552158f6bb091a0437bb/.tmp/colfamily11/073b7584760e43e2994f8057899a6091 is 52, key is ttt/colfamily11:colfamily11/1731736292156/Put/seqid=0 2024-11-16T05:54:39,412 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/de75e055385b7b1b204f0aa9c1924d0e/.tmp/colfamily11/194246ac0fd442c7b87b91d292e9194b is 52, key is ccc/colfamily11:colfamily11/1731736291884/Put/seqid=0 2024-11-16T05:54:39,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741901_1077 (size=39263) 2024-11-16T05:54:39,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741901_1077 (size=39263) 2024-11-16T05:54:39,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741901_1077 (size=39263) 2024-11-16T05:54:39,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741902_1078 (size=39263) 2024-11-16T05:54:39,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741902_1078 (size=39263) 2024-11-16T05:54:39,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741902_1078 (size=39263) 2024-11-16T05:54:39,429 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/de75e055385b7b1b204f0aa9c1924d0e/.tmp/colfamily11/194246ac0fd442c7b87b91d292e9194b 2024-11-16T05:54:39,437 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/de75e055385b7b1b204f0aa9c1924d0e/.tmp/colfamily11/194246ac0fd442c7b87b91d292e9194b as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/de75e055385b7b1b204f0aa9c1924d0e/colfamily11/194246ac0fd442c7b87b91d292e9194b 2024-11-16T05:54:39,450 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/de75e055385b7b1b204f0aa9c1924d0e/colfamily11/194246ac0fd442c7b87b91d292e9194b, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T05:54:39,451 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for de75e055385b7b1b204f0aa9c1924d0e in 60ms, sequenceid=5, compaction requested=false 2024-11-16T05:54:39,456 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/de75e055385b7b1b204f0aa9c1924d0e/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T05:54:39,457 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731736288173.de75e055385b7b1b204f0aa9c1924d0e. 2024-11-16T05:54:39,457 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for de75e055385b7b1b204f0aa9c1924d0e: Waiting for close lock at 1731736479391Running coprocessor pre-close hooks at 1731736479391Disabling compacts and flushes for region at 1731736479391Disabling writes for close at 1731736479391Obtaining lock to block concurrent updates at 1731736479391Preparing flush snapshotting stores in de75e055385b7b1b204f0aa9c1924d0e at 1731736479391Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731736288173.de75e055385b7b1b204f0aa9c1924d0e., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731736479392 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731736288173.de75e055385b7b1b204f0aa9c1924d0e. at 1731736479392Flushing de75e055385b7b1b204f0aa9c1924d0e/colfamily11: creating writer at 1731736479392Flushing de75e055385b7b1b204f0aa9c1924d0e/colfamily11: appending metadata at 1731736479412 (+20 ms)Flushing de75e055385b7b1b204f0aa9c1924d0e/colfamily11: closing flushed file at 1731736479412Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43eaf338: reopening flushed file at 1731736479436 (+24 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for de75e055385b7b1b204f0aa9c1924d0e in 60ms, sequenceid=5, compaction requested=false at 1731736479451 (+15 ms)Writing region close event to WAL at 1731736479452 (+1 ms)Running coprocessor post-close hooks at 1731736479457 (+5 ms)Closed at 1731736479457 2024-11-16T05:54:39,458 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731736288173.de75e055385b7b1b204f0aa9c1924d0e. 2024-11-16T05:54:39,562 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-16T05:54:39,562 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer(976): stopping server 3456ee6a3164,33007,1731736285465; all regions closed. 2024-11-16T05:54:39,562 DEBUG [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(1325): Online Regions={2beaf8caee4e552158f6bb091a0437bb=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731736288173.2beaf8caee4e552158f6bb091a0437bb., afd4d1a884eb847b35b730a1b7a90e0d=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731736288173.afd4d1a884eb847b35b730a1b7a90e0d.} 2024-11-16T05:54:39,562 DEBUG [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(1351): Waiting on 2beaf8caee4e552158f6bb091a0437bb, afd4d1a884eb847b35b730a1b7a90e0d 2024-11-16T05:54:39,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741834_1010 (size=316519) 2024-11-16T05:54:39,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741834_1010 (size=316519) 2024-11-16T05:54:39,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741834_1010 (size=316519) 2024-11-16T05:54:39,568 DEBUG [RS:0;3456ee6a3164:33007 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/oldWALs 2024-11-16T05:54:39,569 INFO [RS:0;3456ee6a3164:33007 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3456ee6a3164%2C33007%2C1731736285465:(num 1731736287193) 2024-11-16T05:54:39,569 DEBUG [RS:0;3456ee6a3164:33007 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:54:39,569 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T05:54:39,569 INFO [RS:0;3456ee6a3164:33007 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T05:54:39,569 INFO [RS:0;3456ee6a3164:33007 {}] hbase.ChoreService(370): Chore service for: regionserver/3456ee6a3164:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T05:54:39,569 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T05:54:39,570 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T05:54:39,570 INFO [regionserver/3456ee6a3164:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T05:54:39,570 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T05:54:39,570 INFO [RS:0;3456ee6a3164:33007 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T05:54:39,570 INFO [RS:0;3456ee6a3164:33007 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33007 2024-11-16T05:54:39,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3456ee6a3164,33007,1731736285465 2024-11-16T05:54:39,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T05:54:39,573 INFO [RS:0;3456ee6a3164:33007 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T05:54:39,576 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3456ee6a3164,33007,1731736285465] 2024-11-16T05:54:39,577 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3456ee6a3164,33007,1731736285465 already deleted, retry=false 2024-11-16T05:54:39,577 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 3456ee6a3164,33007,1731736285465 expired; onlineServers=1 2024-11-16T05:54:39,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:54:39,676 INFO [RS:0;3456ee6a3164:33007 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T05:54:39,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33007-0x100471607020001, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:54:39,676 INFO [RS:0;3456ee6a3164:33007 {}] regionserver.HRegionServer(1031): Exiting; stopping=3456ee6a3164,33007,1731736285465; zookeeper connection closed. 2024-11-16T05:54:39,677 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2ceb6dde {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2ceb6dde 2024-11-16T05:54:39,762 DEBUG [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(1351): Waiting on 2beaf8caee4e552158f6bb091a0437bb, afd4d1a884eb847b35b730a1b7a90e0d 2024-11-16T05:54:39,788 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=3456ee6a3164:44685 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-16T05:54:39,821 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2beaf8caee4e552158f6bb091a0437bb/.tmp/colfamily11/073b7584760e43e2994f8057899a6091 2024-11-16T05:54:39,832 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2beaf8caee4e552158f6bb091a0437bb/.tmp/colfamily11/073b7584760e43e2994f8057899a6091 as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2beaf8caee4e552158f6bb091a0437bb/colfamily11/073b7584760e43e2994f8057899a6091 2024-11-16T05:54:39,838 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2beaf8caee4e552158f6bb091a0437bb/colfamily11/073b7584760e43e2994f8057899a6091, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T05:54:39,839 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 2beaf8caee4e552158f6bb091a0437bb in 468ms, sequenceid=5, compaction requested=false 2024-11-16T05:54:39,843 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2beaf8caee4e552158f6bb091a0437bb/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T05:54:39,844 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731736288173.2beaf8caee4e552158f6bb091a0437bb. 2024-11-16T05:54:39,844 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2beaf8caee4e552158f6bb091a0437bb: Waiting for close lock at 1731736479371Running coprocessor pre-close hooks at 1731736479371Disabling compacts and flushes for region at 1731736479371Disabling writes for close at 1731736479371Obtaining lock to block concurrent updates at 1731736479371Preparing flush snapshotting stores in 2beaf8caee4e552158f6bb091a0437bb at 1731736479371Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731736288173.2beaf8caee4e552158f6bb091a0437bb., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731736479372 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731736288173.2beaf8caee4e552158f6bb091a0437bb. at 1731736479372Flushing 2beaf8caee4e552158f6bb091a0437bb/colfamily11: creating writer at 1731736479372Flushing 2beaf8caee4e552158f6bb091a0437bb/colfamily11: appending metadata at 1731736479401 (+29 ms)Flushing 2beaf8caee4e552158f6bb091a0437bb/colfamily11: closing flushed file at 1731736479402 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3dc66f45: reopening flushed file at 1731736479831 (+429 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 2beaf8caee4e552158f6bb091a0437bb in 468ms, sequenceid=5, compaction requested=false at 1731736479839 (+8 ms)Writing region close event to WAL at 1731736479840 (+1 ms)Running coprocessor post-close hooks at 1731736479844 (+4 ms)Closed at 1731736479844 2024-11-16T05:54:39,844 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731736288173.2beaf8caee4e552158f6bb091a0437bb. 2024-11-16T05:54:39,844 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing afd4d1a884eb847b35b730a1b7a90e0d, disabling compactions & flushes 2024-11-16T05:54:39,844 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731736288173.afd4d1a884eb847b35b730a1b7a90e0d. 2024-11-16T05:54:39,844 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731736288173.afd4d1a884eb847b35b730a1b7a90e0d. 2024-11-16T05:54:39,844 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731736288173.afd4d1a884eb847b35b730a1b7a90e0d. after waiting 0 ms 2024-11-16T05:54:39,844 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731736288173.afd4d1a884eb847b35b730a1b7a90e0d. 2024-11-16T05:54:39,844 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing afd4d1a884eb847b35b730a1b7a90e0d 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-16T05:54:39,859 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/afd4d1a884eb847b35b730a1b7a90e0d/.tmp/colfamily11/0680c7ed5d94424bbc5348d5124a0d9f is 52, key is yyy/colfamily11:colfamily11/1731736292197/Put/seqid=0 2024-11-16T05:54:39,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741903_1079 (size=39263) 2024-11-16T05:54:39,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741903_1079 (size=39263) 2024-11-16T05:54:39,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741903_1079 (size=39263) 2024-11-16T05:54:39,866 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/afd4d1a884eb847b35b730a1b7a90e0d/.tmp/colfamily11/0680c7ed5d94424bbc5348d5124a0d9f 2024-11-16T05:54:39,872 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/afd4d1a884eb847b35b730a1b7a90e0d/.tmp/colfamily11/0680c7ed5d94424bbc5348d5124a0d9f as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/afd4d1a884eb847b35b730a1b7a90e0d/colfamily11/0680c7ed5d94424bbc5348d5124a0d9f 2024-11-16T05:54:39,878 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/afd4d1a884eb847b35b730a1b7a90e0d/colfamily11/0680c7ed5d94424bbc5348d5124a0d9f, entries=703, sequenceid=5, filesize=38.3 K 2024-11-16T05:54:39,878 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for afd4d1a884eb847b35b730a1b7a90e0d in 34ms, sequenceid=5, compaction requested=false 2024-11-16T05:54:39,882 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/afd4d1a884eb847b35b730a1b7a90e0d/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-16T05:54:39,883 INFO [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731736288173.afd4d1a884eb847b35b730a1b7a90e0d. 2024-11-16T05:54:39,883 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for afd4d1a884eb847b35b730a1b7a90e0d: Waiting for close lock at 1731736479844Running coprocessor pre-close hooks at 1731736479844Disabling compacts and flushes for region at 1731736479844Disabling writes for close at 1731736479844Obtaining lock to block concurrent updates at 1731736479844Preparing flush snapshotting stores in afd4d1a884eb847b35b730a1b7a90e0d at 1731736479844Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731736288173.afd4d1a884eb847b35b730a1b7a90e0d., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731736479845 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731736288173.afd4d1a884eb847b35b730a1b7a90e0d. at 1731736479845Flushing afd4d1a884eb847b35b730a1b7a90e0d/colfamily11: creating writer at 1731736479845Flushing afd4d1a884eb847b35b730a1b7a90e0d/colfamily11: appending metadata at 1731736479858 (+13 ms)Flushing afd4d1a884eb847b35b730a1b7a90e0d/colfamily11: closing flushed file at 1731736479858Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d074bc4: reopening flushed file at 1731736479871 (+13 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for afd4d1a884eb847b35b730a1b7a90e0d in 34ms, sequenceid=5, compaction requested=false at 1731736479878 (+7 ms)Writing region close event to WAL at 1731736479879 (+1 ms)Running coprocessor post-close hooks at 1731736479883 (+4 ms)Closed at 1731736479883 2024-11-16T05:54:39,883 DEBUG [RS_CLOSE_REGION-regionserver/3456ee6a3164:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731736288173.afd4d1a884eb847b35b730a1b7a90e0d. 2024-11-16T05:54:39,963 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(976): stopping server 3456ee6a3164,43415,1731736285705; all regions closed. 2024-11-16T05:54:39,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741865_1041 (size=14124) 2024-11-16T05:54:39,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741865_1041 (size=14124) 2024-11-16T05:54:39,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741865_1041 (size=14124) 2024-11-16T05:54:39,968 DEBUG [RS:2;3456ee6a3164:43415 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/oldWALs 2024-11-16T05:54:39,968 INFO [RS:2;3456ee6a3164:43415 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3456ee6a3164%2C43415%2C1731736285705.meta:.meta(num 1731736473355) 2024-11-16T05:54:39,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741833_1009 (size=353044) 2024-11-16T05:54:39,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741833_1009 (size=353044) 2024-11-16T05:54:39,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741833_1009 (size=353044) 2024-11-16T05:54:39,972 DEBUG [RS:2;3456ee6a3164:43415 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/oldWALs 2024-11-16T05:54:39,972 INFO [RS:2;3456ee6a3164:43415 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3456ee6a3164%2C43415%2C1731736285705:(num 1731736287193) 2024-11-16T05:54:39,972 DEBUG [RS:2;3456ee6a3164:43415 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T05:54:39,972 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T05:54:39,972 INFO [RS:2;3456ee6a3164:43415 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T05:54:39,973 INFO [RS:2;3456ee6a3164:43415 {}] hbase.ChoreService(370): Chore service for: regionserver/3456ee6a3164:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T05:54:39,973 INFO [RS:2;3456ee6a3164:43415 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T05:54:39,973 INFO [regionserver/3456ee6a3164:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T05:54:39,973 INFO [RS:2;3456ee6a3164:43415 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43415 2024-11-16T05:54:39,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3456ee6a3164,43415,1731736285705 2024-11-16T05:54:39,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T05:54:39,975 INFO [RS:2;3456ee6a3164:43415 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T05:54:39,979 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3456ee6a3164,43415,1731736285705] 2024-11-16T05:54:39,982 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3456ee6a3164,43415,1731736285705 already deleted, retry=false 2024-11-16T05:54:39,982 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 3456ee6a3164,43415,1731736285705 expired; onlineServers=0 2024-11-16T05:54:39,982 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '3456ee6a3164,44685,1731736284655' ***** 2024-11-16T05:54:39,982 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-16T05:54:39,982 INFO [M:0;3456ee6a3164:44685 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T05:54:39,983 INFO [M:0;3456ee6a3164:44685 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T05:54:39,983 DEBUG [M:0;3456ee6a3164:44685 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T05:54:39,983 DEBUG [M:0;3456ee6a3164:44685 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T05:54:39,983 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T05:54:39,983 DEBUG [master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.large.0-1731736286842 {}] cleaner.HFileCleaner(306): Exit Thread[master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.large.0-1731736286842,5,FailOnTimeoutGroup] 2024-11-16T05:54:39,983 DEBUG [master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.small.0-1731736286853 {}] cleaner.HFileCleaner(306): Exit Thread[master/3456ee6a3164:0:becomeActiveMaster-HFileCleaner.small.0-1731736286853,5,FailOnTimeoutGroup] 2024-11-16T05:54:39,983 INFO [M:0;3456ee6a3164:44685 {}] hbase.ChoreService(370): Chore service for: master/3456ee6a3164:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T05:54:39,983 INFO [M:0;3456ee6a3164:44685 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T05:54:39,983 DEBUG [M:0;3456ee6a3164:44685 {}] master.HMaster(1795): Stopping service threads 2024-11-16T05:54:39,983 INFO [M:0;3456ee6a3164:44685 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T05:54:39,983 INFO [M:0;3456ee6a3164:44685 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T05:54:39,984 INFO [M:0;3456ee6a3164:44685 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T05:54:39,984 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T05:54:39,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T05:54:39,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T05:54:39,984 DEBUG [M:0;3456ee6a3164:44685 {}] zookeeper.ZKUtil(347): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-16T05:54:39,984 WARN [M:0;3456ee6a3164:44685 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-16T05:54:39,985 INFO [M:0;3456ee6a3164:44685 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/.lastflushedseqids 2024-11-16T05:54:39,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741904_1080 (size=2864) 2024-11-16T05:54:39,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741904_1080 (size=2864) 2024-11-16T05:54:39,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741904_1080 (size=2864) 2024-11-16T05:54:39,998 INFO [M:0;3456ee6a3164:44685 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T05:54:39,998 INFO [M:0;3456ee6a3164:44685 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-16T05:54:39,998 DEBUG [M:0;3456ee6a3164:44685 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T05:54:39,998 INFO [M:0;3456ee6a3164:44685 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:54:39,998 DEBUG [M:0;3456ee6a3164:44685 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:54:39,998 DEBUG [M:0;3456ee6a3164:44685 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T05:54:39,998 DEBUG [M:0;3456ee6a3164:44685 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:54:39,999 INFO [M:0;3456ee6a3164:44685 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=336.42 KB heapSize=395.41 KB 2024-11-16T05:54:40,019 DEBUG [M:0;3456ee6a3164:44685 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b326fa672f8a40679209e41f91182d25 is 82, key is hbase:meta,,1/info:regioninfo/1731736473842/Put/seqid=0 2024-11-16T05:54:40,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741905_1081 (size=6071) 2024-11-16T05:54:40,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741905_1081 (size=6071) 2024-11-16T05:54:40,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741905_1081 (size=6071) 2024-11-16T05:54:40,026 INFO [M:0;3456ee6a3164:44685 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=620 (bloomFilter=true), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b326fa672f8a40679209e41f91182d25 2024-11-16T05:54:40,045 DEBUG [M:0;3456ee6a3164:44685 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3024f5e8773e471cbc47a9323020656f is 4106, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731736290216/Put/seqid=0 2024-11-16T05:54:40,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741906_1082 (size=13327) 2024-11-16T05:54:40,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741906_1082 (size=13327) 2024-11-16T05:54:40,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741906_1082 (size=13327) 2024-11-16T05:54:40,051 INFO [M:0;3456ee6a3164:44685 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=335.19 KB at sequenceid=620 (bloomFilter=true), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3024f5e8773e471cbc47a9323020656f 2024-11-16T05:54:40,072 DEBUG [M:0;3456ee6a3164:44685 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/894a5438a1df442ab8a46c62d10c6618 is 69, key is 3456ee6a3164,33007,1731736285465/rs:state/1731736286921/Put/seqid=0 2024-11-16T05:54:40,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741907_1083 (size=5442) 2024-11-16T05:54:40,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741907_1083 (size=5442) 2024-11-16T05:54:40,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741907_1083 (size=5442) 2024-11-16T05:54:40,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:54:40,078 INFO [RS:2;3456ee6a3164:43415 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T05:54:40,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43415-0x100471607020003, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:54:40,078 INFO [RS:2;3456ee6a3164:43415 {}] regionserver.HRegionServer(1031): Exiting; stopping=3456ee6a3164,43415,1731736285705; zookeeper connection closed. 2024-11-16T05:54:40,078 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@78050323 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@78050323 2024-11-16T05:54:40,079 INFO [M:0;3456ee6a3164:44685 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=620 (bloomFilter=true), to=hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/894a5438a1df442ab8a46c62d10c6618 2024-11-16T05:54:40,079 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-16T05:54:40,083 INFO [M:0;3456ee6a3164:44685 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 894a5438a1df442ab8a46c62d10c6618 2024-11-16T05:54:40,084 DEBUG [M:0;3456ee6a3164:44685 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b326fa672f8a40679209e41f91182d25 as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b326fa672f8a40679209e41f91182d25 2024-11-16T05:54:40,090 INFO [M:0;3456ee6a3164:44685 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b326fa672f8a40679209e41f91182d25, entries=14, sequenceid=620, filesize=5.9 K 2024-11-16T05:54:40,091 DEBUG [M:0;3456ee6a3164:44685 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3024f5e8773e471cbc47a9323020656f as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3024f5e8773e471cbc47a9323020656f 2024-11-16T05:54:40,098 INFO [M:0;3456ee6a3164:44685 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3024f5e8773e471cbc47a9323020656f, entries=79, sequenceid=620, filesize=13.0 K 2024-11-16T05:54:40,099 DEBUG [M:0;3456ee6a3164:44685 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/894a5438a1df442ab8a46c62d10c6618 as hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/894a5438a1df442ab8a46c62d10c6618 2024-11-16T05:54:40,105 INFO [M:0;3456ee6a3164:44685 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 894a5438a1df442ab8a46c62d10c6618 2024-11-16T05:54:40,105 INFO [M:0;3456ee6a3164:44685 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45275/user/jenkins/test-data/96041237-b202-1a34-0fcb-1684e0eca8f8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/894a5438a1df442ab8a46c62d10c6618, entries=3, sequenceid=620, filesize=5.3 K 2024-11-16T05:54:40,106 INFO [M:0;3456ee6a3164:44685 {}] regionserver.HRegion(3140): Finished flush of dataSize ~336.42 KB/344489, heapSize ~395.12 KB/404600, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 108ms, sequenceid=620, compaction requested=false 2024-11-16T05:54:40,108 INFO [M:0;3456ee6a3164:44685 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T05:54:40,108 DEBUG [M:0;3456ee6a3164:44685 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731736479998Disabling compacts and flushes for region at 1731736479998Disabling writes for close at 1731736479998Obtaining lock to block concurrent updates at 1731736479999 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731736479999Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=344489, getHeapSize=404840, getOffHeapSize=0, getCellsCount=781 at 1731736479999Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731736480000 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731736480000Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731736480019 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731736480019Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731736480031 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731736480045 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731736480045Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731736480056 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731736480071 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731736480071Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14f5332c: reopening flushed file at 1731736480083 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@240033cb: reopening flushed file at 1731736480090 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d797da6: reopening flushed file at 1731736480098 (+8 ms)Finished flush of dataSize ~336.42 KB/344489, heapSize ~395.12 KB/404600, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 108ms, sequenceid=620, compaction requested=false at 1731736480106 (+8 ms)Writing region close event to WAL at 1731736480107 (+1 ms)Closed at 1731736480108 (+1 ms) 2024-11-16T05:54:40,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36993 is added to blk_1073741830_1006 (size=385852) 2024-11-16T05:54:40,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32873 is added to blk_1073741830_1006 (size=385852) 2024-11-16T05:54:40,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33441 is added to blk_1073741830_1006 (size=385852) 2024-11-16T05:54:40,111 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T05:54:40,111 INFO [M:0;3456ee6a3164:44685 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-16T05:54:40,112 INFO [M:0;3456ee6a3164:44685 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44685 2024-11-16T05:54:40,112 INFO [M:0;3456ee6a3164:44685 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T05:54:40,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:54:40,215 INFO [M:0;3456ee6a3164:44685 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T05:54:40,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44685-0x100471607020000, quorum=127.0.0.1:58523, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T05:54:40,224 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@22333f25{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T05:54:40,227 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@eec57c1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T05:54:40,227 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T05:54:40,227 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@17888407{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T05:54:40,227 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@30067af9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/hadoop.log.dir/,STOPPED} 2024-11-16T05:54:40,229 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-16T05:54:40,229 WARN [BP-1182044883-172.17.0.2-1731736281266 heartbeating to localhost/127.0.0.1:45275 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-16T05:54:40,229 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-16T05:54:40,229 WARN [BP-1182044883-172.17.0.2-1731736281266 heartbeating to localhost/127.0.0.1:45275 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1182044883-172.17.0.2-1731736281266 (Datanode Uuid 63e24c74-3be6-46bf-ad1a-7f46f1bfb235) service to localhost/127.0.0.1:45275 2024-11-16T05:54:40,230 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/cluster_59495369-691a-be10-1b61-6f414eddee7f/data/data5/current/BP-1182044883-172.17.0.2-1731736281266 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T05:54:40,230 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/cluster_59495369-691a-be10-1b61-6f414eddee7f/data/data6/current/BP-1182044883-172.17.0.2-1731736281266 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T05:54:40,231 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-16T05:54:40,233 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1ba455be{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T05:54:40,233 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@259771a6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T05:54:40,233 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T05:54:40,233 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6c516a28{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T05:54:40,233 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7f4017e5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/hadoop.log.dir/,STOPPED} 2024-11-16T05:54:40,234 WARN [BP-1182044883-172.17.0.2-1731736281266 heartbeating to localhost/127.0.0.1:45275 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-16T05:54:40,234 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-16T05:54:40,234 WARN [BP-1182044883-172.17.0.2-1731736281266 heartbeating to localhost/127.0.0.1:45275 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1182044883-172.17.0.2-1731736281266 (Datanode Uuid c8992440-ff96-4231-ad71-71d7f7ce1556) service to localhost/127.0.0.1:45275 2024-11-16T05:54:40,234 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-16T05:54:40,235 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/cluster_59495369-691a-be10-1b61-6f414eddee7f/data/data3/current/BP-1182044883-172.17.0.2-1731736281266 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T05:54:40,235 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/cluster_59495369-691a-be10-1b61-6f414eddee7f/data/data4/current/BP-1182044883-172.17.0.2-1731736281266 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T05:54:40,235 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-16T05:54:40,238 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4d81bf0e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T05:54:40,238 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@572fe942{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T05:54:40,239 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T05:54:40,239 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@664a7783{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T05:54:40,239 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7da31712{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/hadoop.log.dir/,STOPPED} 2024-11-16T05:54:40,240 WARN [BP-1182044883-172.17.0.2-1731736281266 heartbeating to localhost/127.0.0.1:45275 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-16T05:54:40,240 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-16T05:54:40,240 WARN [BP-1182044883-172.17.0.2-1731736281266 heartbeating to localhost/127.0.0.1:45275 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1182044883-172.17.0.2-1731736281266 (Datanode Uuid c5d2e700-f00c-4f38-96af-fd6fb0055b8a) service to localhost/127.0.0.1:45275 2024-11-16T05:54:40,240 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-16T05:54:40,241 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/cluster_59495369-691a-be10-1b61-6f414eddee7f/data/data1/current/BP-1182044883-172.17.0.2-1731736281266 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T05:54:40,241 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/cluster_59495369-691a-be10-1b61-6f414eddee7f/data/data2/current/BP-1182044883-172.17.0.2-1731736281266 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T05:54:40,241 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-16T05:54:40,250 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5183f126{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T05:54:40,250 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4350931a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T05:54:40,250 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T05:54:40,251 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4974766f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T05:54:40,251 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2c447bb0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f9a05250-4423-0d7e-b6c7-1c7bf018478d/hadoop.log.dir/,STOPPED} 2024-11-16T05:54:40,264 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-16T05:54:40,325 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-16T05:54:40,334 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.procedure.TestSCPWithMeta#testRecoveryAndDoubleExecutionOnRsWithMeta Thread=89 (was 11) Potentially hanging thread: regionserver/3456ee6a3164:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@29b7a5ea java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Abort regionserver monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: SSL Certificates Store Monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45275 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/3456ee6a3164:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45275 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:45275 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (102774891) connection to localhost/127.0.0.1:45275 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45275 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:45275 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (102774891) connection to localhost/127.0.0.1:45275 from jenkins.hfs.2 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: regionserver/3456ee6a3164:0.Chore.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/3456ee6a3164:0.logRoller java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:180) Potentially hanging thread: nioEventLoopGroup-7-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45275 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/3456ee6a3164:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (102774891) connection to localhost/127.0.0.1:45275 from jenkins.hfs.0 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: nioEventLoopGroup-2-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161) Potentially hanging thread: nioEventLoopGroup-2-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: nioEventLoopGroup-3-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-7-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: BootstrapNodeManager java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (102774891) connection to localhost/127.0.0.1:45275 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-6-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-6-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/3456ee6a3164:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/3456ee6a3164:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-7-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_META-regionserver/3456ee6a3164:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-6-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/3456ee6a3164:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:45275 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=454 (was 285) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 200), ProcessCount=11 (was 11), AvailableMemoryMB=3133 (was 4730)