2024-11-12 14:49:19,562 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-12 14:49:19,587 main DEBUG Took 0.022901 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-12 14:49:19,587 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-12 14:49:19,588 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-12 14:49:19,589 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-12 14:49:19,590 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:49:19,598 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-12 14:49:19,611 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:49:19,612 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:49:19,613 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:49:19,613 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:49:19,614 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:49:19,614 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:49:19,615 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:49:19,615 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:49:19,616 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:49:19,616 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:49:19,617 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:49:19,617 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:49:19,618 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:49:19,618 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:49:19,619 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:49:19,619 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:49:19,619 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:49:19,620 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:49:19,620 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:49:19,620 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:49:19,621 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:49:19,621 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:49:19,622 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:49:19,622 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:49:19,622 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:49:19,623 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-12 14:49:19,624 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:49:19,625 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-12 14:49:19,627 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-12 14:49:19,628 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-12 14:49:19,630 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-12 14:49:19,630 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-12 14:49:19,641 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-12 14:49:19,644 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-12 14:49:19,647 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-12 14:49:19,647 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-12 14:49:19,648 main DEBUG createAppenders(={Console}) 2024-11-12 14:49:19,649 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-12 14:49:19,649 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-12 14:49:19,650 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-12 14:49:19,650 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-12 14:49:19,651 main DEBUG OutputStream closed 2024-11-12 14:49:19,651 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-12 14:49:19,651 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-12 14:49:19,652 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-12 14:49:19,740 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-12 14:49:19,743 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-12 14:49:19,744 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-12 14:49:19,745 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-12 14:49:19,746 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-12 14:49:19,746 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-12 14:49:19,747 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-12 14:49:19,747 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-12 14:49:19,747 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-12 14:49:19,748 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-12 14:49:19,748 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-12 14:49:19,748 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-12 14:49:19,749 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-12 14:49:19,749 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-12 14:49:19,749 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-12 14:49:19,749 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-12 14:49:19,750 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-12 14:49:19,750 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-12 14:49:19,752 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12 14:49:19,752 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-12 14:49:19,753 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-12 14:49:19,753 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-12T14:49:19,765 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta timeout: 13 mins 2024-11-12 14:49:19,767 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-12 14:49:19,768 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12T14:49:19,803 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.procedure.TestSCPWithMeta#testRecoveryAndDoubleExecutionOnRsWithMeta Thread=11, OpenFileDescriptor=285, MaxFileDescriptor=1048576, SystemLoadAverage=149, ProcessCount=11, AvailableMemoryMB=8848 2024-11-12T14:49:20,132 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c 2024-11-12T14:49:20,133 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-12T14:49:20,150 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/cluster_fa7c45b8-f5f8-b08a-b595-b8fe56346db2, deleteOnExit=true 2024-11-12T14:49:20,151 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-12T14:49:20,152 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/test.cache.data in system properties and HBase conf 2024-11-12T14:49:20,152 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/hadoop.tmp.dir in system properties and HBase conf 2024-11-12T14:49:20,153 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/hadoop.log.dir in system properties and HBase conf 2024-11-12T14:49:20,154 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-12T14:49:20,154 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-12T14:49:20,155 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-12T14:49:20,230 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-12T14:49:20,327 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T14:49:20,331 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-12T14:49:20,332 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-12T14:49:20,333 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-12T14:49:20,334 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T14:49:20,334 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-12T14:49:20,335 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-12T14:49:20,336 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T14:49:20,336 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T14:49:20,337 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-12T14:49:20,338 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/nfs.dump.dir in system properties and HBase conf 2024-11-12T14:49:20,338 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/java.io.tmpdir in system properties and HBase conf 2024-11-12T14:49:20,339 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T14:49:20,339 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-12T14:49:20,340 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-12T14:49:21,115 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-12T14:49:21,188 INFO [Time-limited test {}] log.Log(170): Logging initialized @2284ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-12T14:49:21,267 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T14:49:21,335 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T14:49:21,363 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T14:49:21,363 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T14:49:21,365 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-12T14:49:21,379 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T14:49:21,382 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@58897554{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/hadoop.log.dir/,AVAILABLE} 2024-11-12T14:49:21,383 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2a372edd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T14:49:21,578 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1aa62678{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/java.io.tmpdir/jetty-localhost-38625-hadoop-hdfs-3_4_1-tests_jar-_-any-16495076751705285213/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T14:49:21,586 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1c0c42ad{HTTP/1.1, (http/1.1)}{localhost:38625} 2024-11-12T14:49:21,586 INFO [Time-limited test {}] server.Server(415): Started @2683ms 2024-11-12T14:49:21,899 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T14:49:21,907 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T14:49:21,908 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T14:49:21,908 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T14:49:21,908 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-12T14:49:21,909 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b71ffd0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/hadoop.log.dir/,AVAILABLE} 2024-11-12T14:49:21,909 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@63d82da9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T14:49:22,007 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4255463d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/java.io.tmpdir/jetty-localhost-34563-hadoop-hdfs-3_4_1-tests_jar-_-any-14054134937948457392/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T14:49:22,007 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5e48d68a{HTTP/1.1, (http/1.1)}{localhost:34563} 2024-11-12T14:49:22,008 INFO [Time-limited test {}] server.Server(415): Started @3104ms 2024-11-12T14:49:22,055 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-12T14:49:22,180 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T14:49:22,190 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T14:49:22,192 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T14:49:22,192 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T14:49:22,193 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-12T14:49:22,198 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@63779849{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/hadoop.log.dir/,AVAILABLE} 2024-11-12T14:49:22,199 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a292a59{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T14:49:22,331 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3635c31e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/java.io.tmpdir/jetty-localhost-46371-hadoop-hdfs-3_4_1-tests_jar-_-any-5758651682265898225/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T14:49:22,332 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@78839a64{HTTP/1.1, (http/1.1)}{localhost:46371} 2024-11-12T14:49:22,332 INFO [Time-limited test {}] server.Server(415): Started @3429ms 2024-11-12T14:49:22,335 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-12T14:49:22,385 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T14:49:22,391 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T14:49:22,392 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T14:49:22,392 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T14:49:22,392 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-12T14:49:22,393 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ed2891b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/hadoop.log.dir/,AVAILABLE} 2024-11-12T14:49:22,394 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@43bf9dc1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T14:49:22,475 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/cluster_fa7c45b8-f5f8-b08a-b595-b8fe56346db2/data/data2/current/BP-619972518-172.17.0.2-1731422960904/current, will proceed with Du for space computation calculation, 2024-11-12T14:49:22,475 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/cluster_fa7c45b8-f5f8-b08a-b595-b8fe56346db2/data/data3/current/BP-619972518-172.17.0.2-1731422960904/current, will proceed with Du for space computation calculation, 2024-11-12T14:49:22,475 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/cluster_fa7c45b8-f5f8-b08a-b595-b8fe56346db2/data/data4/current/BP-619972518-172.17.0.2-1731422960904/current, will proceed with Du for space computation calculation, 2024-11-12T14:49:22,475 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/cluster_fa7c45b8-f5f8-b08a-b595-b8fe56346db2/data/data1/current/BP-619972518-172.17.0.2-1731422960904/current, will proceed with Du for space computation calculation, 2024-11-12T14:49:22,502 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@486c2677{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/java.io.tmpdir/jetty-localhost-34373-hadoop-hdfs-3_4_1-tests_jar-_-any-17370378115474485258/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T14:49:22,503 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6738d157{HTTP/1.1, (http/1.1)}{localhost:34373} 2024-11-12T14:49:22,503 INFO [Time-limited test {}] server.Server(415): Started @3600ms 2024-11-12T14:49:22,506 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-12T14:49:22,516 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-12T14:49:22,516 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-12T14:49:22,575 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x645d3ffc310c25fc with lease ID 0xf2742d20d853d3aa: Processing first storage report for DS-48b2b78b-5966-4c15-86cf-f894eccf2d5b from datanode DatanodeRegistration(127.0.0.1:44401, datanodeUuid=087d5b94-7f1d-46ba-8026-5884f8bf8180, infoPort=38447, infoSecurePort=0, ipcPort=39783, storageInfo=lv=-57;cid=testClusterID;nsid=795347441;c=1731422960904) 2024-11-12T14:49:22,576 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x645d3ffc310c25fc with lease ID 0xf2742d20d853d3aa: from storage DS-48b2b78b-5966-4c15-86cf-f894eccf2d5b node DatanodeRegistration(127.0.0.1:44401, datanodeUuid=087d5b94-7f1d-46ba-8026-5884f8bf8180, infoPort=38447, infoSecurePort=0, ipcPort=39783, storageInfo=lv=-57;cid=testClusterID;nsid=795347441;c=1731422960904), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-12T14:49:22,577 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6295e3905b638ac7 with lease ID 0xf2742d20d853d3ab: Processing first storage report for DS-2fd665aa-69eb-44fd-bca0-32d25d7b0e8b from datanode DatanodeRegistration(127.0.0.1:38947, datanodeUuid=d624de6c-b131-493c-901a-01994b4c38f0, infoPort=34963, infoSecurePort=0, ipcPort=40783, storageInfo=lv=-57;cid=testClusterID;nsid=795347441;c=1731422960904) 2024-11-12T14:49:22,577 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6295e3905b638ac7 with lease ID 0xf2742d20d853d3ab: from storage DS-2fd665aa-69eb-44fd-bca0-32d25d7b0e8b node DatanodeRegistration(127.0.0.1:38947, datanodeUuid=d624de6c-b131-493c-901a-01994b4c38f0, infoPort=34963, infoSecurePort=0, ipcPort=40783, storageInfo=lv=-57;cid=testClusterID;nsid=795347441;c=1731422960904), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-12T14:49:22,577 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x645d3ffc310c25fc with lease ID 0xf2742d20d853d3aa: Processing first storage report for DS-62e28b6b-a5ab-4d94-b792-6793e95cce27 from datanode DatanodeRegistration(127.0.0.1:44401, datanodeUuid=087d5b94-7f1d-46ba-8026-5884f8bf8180, infoPort=38447, infoSecurePort=0, ipcPort=39783, storageInfo=lv=-57;cid=testClusterID;nsid=795347441;c=1731422960904) 2024-11-12T14:49:22,577 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x645d3ffc310c25fc with lease ID 0xf2742d20d853d3aa: from storage DS-62e28b6b-a5ab-4d94-b792-6793e95cce27 node DatanodeRegistration(127.0.0.1:44401, datanodeUuid=087d5b94-7f1d-46ba-8026-5884f8bf8180, infoPort=38447, infoSecurePort=0, ipcPort=39783, storageInfo=lv=-57;cid=testClusterID;nsid=795347441;c=1731422960904), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-12T14:49:22,578 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6295e3905b638ac7 with lease ID 0xf2742d20d853d3ab: Processing first storage report for DS-8d29fe0e-ef04-4f6e-a9fc-fafbf93a55a2 from datanode DatanodeRegistration(127.0.0.1:38947, datanodeUuid=d624de6c-b131-493c-901a-01994b4c38f0, infoPort=34963, infoSecurePort=0, ipcPort=40783, storageInfo=lv=-57;cid=testClusterID;nsid=795347441;c=1731422960904) 2024-11-12T14:49:22,578 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6295e3905b638ac7 with lease ID 0xf2742d20d853d3ab: from storage DS-8d29fe0e-ef04-4f6e-a9fc-fafbf93a55a2 node DatanodeRegistration(127.0.0.1:38947, datanodeUuid=d624de6c-b131-493c-901a-01994b4c38f0, infoPort=34963, infoSecurePort=0, ipcPort=40783, storageInfo=lv=-57;cid=testClusterID;nsid=795347441;c=1731422960904), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-12T14:49:22,593 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/cluster_fa7c45b8-f5f8-b08a-b595-b8fe56346db2/data/data5/current/BP-619972518-172.17.0.2-1731422960904/current, will proceed with Du for space computation calculation, 2024-11-12T14:49:22,593 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/cluster_fa7c45b8-f5f8-b08a-b595-b8fe56346db2/data/data6/current/BP-619972518-172.17.0.2-1731422960904/current, will proceed with Du for space computation calculation, 2024-11-12T14:49:22,620 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-12T14:49:22,625 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xac98b8abc9117325 with lease ID 0xf2742d20d853d3ac: Processing first storage report for DS-7b068bce-ec51-4d06-8fdd-88c861866bd0 from datanode DatanodeRegistration(127.0.0.1:41429, datanodeUuid=db6abbf5-482f-46a9-8114-3d0e0f8fbd8a, infoPort=36551, infoSecurePort=0, ipcPort=34633, storageInfo=lv=-57;cid=testClusterID;nsid=795347441;c=1731422960904) 2024-11-12T14:49:22,625 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xac98b8abc9117325 with lease ID 0xf2742d20d853d3ac: from storage DS-7b068bce-ec51-4d06-8fdd-88c861866bd0 node DatanodeRegistration(127.0.0.1:41429, datanodeUuid=db6abbf5-482f-46a9-8114-3d0e0f8fbd8a, infoPort=36551, infoSecurePort=0, ipcPort=34633, storageInfo=lv=-57;cid=testClusterID;nsid=795347441;c=1731422960904), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-12T14:49:22,626 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xac98b8abc9117325 with lease ID 0xf2742d20d853d3ac: Processing first storage report for DS-e1a469b3-2057-487a-ad31-ab128db3efd6 from datanode DatanodeRegistration(127.0.0.1:41429, datanodeUuid=db6abbf5-482f-46a9-8114-3d0e0f8fbd8a, infoPort=36551, infoSecurePort=0, ipcPort=34633, storageInfo=lv=-57;cid=testClusterID;nsid=795347441;c=1731422960904) 2024-11-12T14:49:22,626 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xac98b8abc9117325 with lease ID 0xf2742d20d853d3ac: from storage DS-e1a469b3-2057-487a-ad31-ab128db3efd6 node DatanodeRegistration(127.0.0.1:41429, datanodeUuid=db6abbf5-482f-46a9-8114-3d0e0f8fbd8a, infoPort=36551, infoSecurePort=0, ipcPort=34633, storageInfo=lv=-57;cid=testClusterID;nsid=795347441;c=1731422960904), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-12T14:49:22,853 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c 2024-11-12T14:49:22,943 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/cluster_fa7c45b8-f5f8-b08a-b595-b8fe56346db2/zookeeper_0, clientPort=62355, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/cluster_fa7c45b8-f5f8-b08a-b595-b8fe56346db2/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/cluster_fa7c45b8-f5f8-b08a-b595-b8fe56346db2/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-12T14:49:22,956 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62355 2024-11-12T14:49:22,969 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T14:49:22,972 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T14:49:23,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741825_1001 (size=7) 2024-11-12T14:49:23,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741825_1001 (size=7) 2024-11-12T14:49:23,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741825_1001 (size=7) 2024-11-12T14:49:23,605 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 with version=8 2024-11-12T14:49:23,605 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/hbase-staging 2024-11-12T14:49:23,703 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-12T14:49:23,959 INFO [Time-limited test {}] client.ConnectionUtils(128): master/3945e1ca6f47:0 server-side Connection retries=45 2024-11-12T14:49:23,969 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:49:23,970 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T14:49:23,975 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T14:49:23,976 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:49:23,976 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T14:49:24,141 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-12T14:49:24,210 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-12T14:49:24,219 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-12T14:49:24,224 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T14:49:24,259 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 31895 (auto-detected) 2024-11-12T14:49:24,260 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-12T14:49:24,288 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38303 2024-11-12T14:49:24,315 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38303 connecting to ZooKeeper ensemble=127.0.0.1:62355 2024-11-12T14:49:24,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:383030x0, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T14:49:24,359 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38303-0x1003469214f0000 connected 2024-11-12T14:49:24,394 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T14:49:24,399 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T14:49:24,412 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T14:49:24,417 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050, hbase.cluster.distributed=false 2024-11-12T14:49:24,448 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T14:49:24,457 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38303 2024-11-12T14:49:24,460 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38303 2024-11-12T14:49:24,461 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38303 2024-11-12T14:49:24,463 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38303 2024-11-12T14:49:24,463 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38303 2024-11-12T14:49:24,616 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3945e1ca6f47:0 server-side Connection retries=45 2024-11-12T14:49:24,618 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:49:24,619 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T14:49:24,619 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T14:49:24,619 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:49:24,619 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T14:49:24,623 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T14:49:24,627 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T14:49:24,628 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37185 2024-11-12T14:49:24,631 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37185 connecting to ZooKeeper ensemble=127.0.0.1:62355 2024-11-12T14:49:24,632 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T14:49:24,636 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T14:49:24,643 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:371850x0, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T14:49:24,644 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:371850x0, quorum=127.0.0.1:62355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T14:49:24,644 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37185-0x1003469214f0001 connected 2024-11-12T14:49:24,648 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T14:49:24,656 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T14:49:24,658 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37185-0x1003469214f0001, quorum=127.0.0.1:62355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T14:49:24,666 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37185-0x1003469214f0001, quorum=127.0.0.1:62355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T14:49:24,667 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37185 2024-11-12T14:49:24,669 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37185 2024-11-12T14:49:24,670 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37185 2024-11-12T14:49:24,673 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37185 2024-11-12T14:49:24,674 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37185 2024-11-12T14:49:24,695 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3945e1ca6f47:0 server-side Connection retries=45 2024-11-12T14:49:24,695 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:49:24,695 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T14:49:24,696 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T14:49:24,696 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:49:24,696 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T14:49:24,696 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T14:49:24,697 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T14:49:24,698 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45845 2024-11-12T14:49:24,700 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45845 connecting to ZooKeeper ensemble=127.0.0.1:62355 2024-11-12T14:49:24,701 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T14:49:24,704 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T14:49:24,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:458450x0, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T14:49:24,710 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:458450x0, quorum=127.0.0.1:62355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T14:49:24,711 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T14:49:24,713 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45845-0x1003469214f0002 connected 2024-11-12T14:49:24,716 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T14:49:24,718 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T14:49:24,721 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T14:49:24,729 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45845 2024-11-12T14:49:24,732 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45845 2024-11-12T14:49:24,733 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45845 2024-11-12T14:49:24,737 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45845 2024-11-12T14:49:24,741 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45845 2024-11-12T14:49:24,758 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/3945e1ca6f47:0 server-side Connection retries=45 2024-11-12T14:49:24,758 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:49:24,758 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T14:49:24,759 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T14:49:24,759 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:49:24,759 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T14:49:24,759 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T14:49:24,759 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T14:49:24,760 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35473 2024-11-12T14:49:24,762 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35473 connecting to ZooKeeper ensemble=127.0.0.1:62355 2024-11-12T14:49:24,764 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T14:49:24,766 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T14:49:24,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:354730x0, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T14:49:24,777 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:354730x0, quorum=127.0.0.1:62355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T14:49:24,778 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T14:49:24,781 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35473-0x1003469214f0003 connected 2024-11-12T14:49:24,785 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T14:49:24,788 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T14:49:24,796 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T14:49:24,801 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35473 2024-11-12T14:49:24,803 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35473 2024-11-12T14:49:24,807 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35473 2024-11-12T14:49:24,808 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35473 2024-11-12T14:49:24,809 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35473 2024-11-12T14:49:24,834 DEBUG [M:0;3945e1ca6f47:38303 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;3945e1ca6f47:38303 2024-11-12T14:49:24,835 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/3945e1ca6f47,38303,1731422963767 2024-11-12T14:49:24,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37185-0x1003469214f0001, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:49:24,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:49:24,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:49:24,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:49:24,847 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/3945e1ca6f47,38303,1731422963767 2024-11-12T14:49:24,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T14:49:24,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T14:49:24,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37185-0x1003469214f0001, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T14:49:24,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:49:24,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:49:24,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:49:24,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37185-0x1003469214f0001, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:49:24,879 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-12T14:49:24,880 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/3945e1ca6f47,38303,1731422963767 from backup master directory 2024-11-12T14:49:24,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37185-0x1003469214f0001, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:49:24,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/3945e1ca6f47,38303,1731422963767 2024-11-12T14:49:24,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:49:24,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:49:24,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:49:24,885 WARN [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T14:49:24,885 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=3945e1ca6f47,38303,1731422963767 2024-11-12T14:49:24,887 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-12T14:49:24,893 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-12T14:49:24,962 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/hbase.id] with ID: 1fc95813-043e-41c4-8f96-df8b2f6ddc4d 2024-11-12T14:49:24,962 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/.tmp/hbase.id 2024-11-12T14:49:24,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741826_1002 (size=42) 2024-11-12T14:49:24,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741826_1002 (size=42) 2024-11-12T14:49:24,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741826_1002 (size=42) 2024-11-12T14:49:24,989 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/.tmp/hbase.id]:[hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/hbase.id] 2024-11-12T14:49:25,032 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T14:49:25,036 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-12T14:49:25,054 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-12T14:49:25,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:49:25,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37185-0x1003469214f0001, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:49:25,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:49:25,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:49:25,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741827_1003 (size=196) 2024-11-12T14:49:25,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741827_1003 (size=196) 2024-11-12T14:49:25,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741827_1003 (size=196) 2024-11-12T14:49:25,105 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T14:49:25,107 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-12T14:49:25,128 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:49:25,132 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:49:25,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741828_1004 (size=1189) 2024-11-12T14:49:25,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741828_1004 (size=1189) 2024-11-12T14:49:25,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741828_1004 (size=1189) 2024-11-12T14:49:25,198 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/MasterData/data/master/store 2024-11-12T14:49:25,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741829_1005 (size=34) 2024-11-12T14:49:25,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741829_1005 (size=34) 2024-11-12T14:49:25,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741829_1005 (size=34) 2024-11-12T14:49:25,228 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-12T14:49:25,231 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:25,233 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T14:49:25,233 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:49:25,234 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:49:25,236 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T14:49:25,236 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:49:25,236 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:49:25,238 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731422965233Disabling compacts and flushes for region at 1731422965233Disabling writes for close at 1731422965236 (+3 ms)Writing region close event to WAL at 1731422965236Closed at 1731422965236 2024-11-12T14:49:25,240 WARN [master/3945e1ca6f47:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/MasterData/data/master/store/.initializing 2024-11-12T14:49:25,240 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/MasterData/WALs/3945e1ca6f47,38303,1731422963767 2024-11-12T14:49:25,250 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T14:49:25,270 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3945e1ca6f47%2C38303%2C1731422963767, suffix=, logDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/MasterData/WALs/3945e1ca6f47,38303,1731422963767, archiveDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/MasterData/oldWALs, maxLogs=10 2024-11-12T14:49:25,297 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/MasterData/WALs/3945e1ca6f47,38303,1731422963767/3945e1ca6f47%2C38303%2C1731422963767.1731422965275, exclude list is [], retry=0 2024-11-12T14:49:25,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38947,DS-2fd665aa-69eb-44fd-bca0-32d25d7b0e8b,DISK] 2024-11-12T14:49:25,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41429,DS-7b068bce-ec51-4d06-8fdd-88c861866bd0,DISK] 2024-11-12T14:49:25,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44401,DS-48b2b78b-5966-4c15-86cf-f894eccf2d5b,DISK] 2024-11-12T14:49:25,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-12T14:49:25,360 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/MasterData/WALs/3945e1ca6f47,38303,1731422963767/3945e1ca6f47%2C38303%2C1731422963767.1731422965275 2024-11-12T14:49:25,361 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34963:34963),(127.0.0.1/127.0.0.1:38447:38447),(127.0.0.1/127.0.0.1:36551:36551)] 2024-11-12T14:49:25,361 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:49:25,362 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:25,365 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:49:25,367 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:49:25,407 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:49:25,438 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-12T14:49:25,443 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:25,448 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:49:25,449 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:49:25,453 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-12T14:49:25,453 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:25,454 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:25,454 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:49:25,457 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-12T14:49:25,458 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:25,459 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:25,459 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:49:25,463 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-12T14:49:25,463 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:25,465 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:25,465 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:49:25,470 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:49:25,472 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:49:25,478 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:49:25,479 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:49:25,482 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T14:49:25,486 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:49:25,492 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:49:25,494 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64525961, jitterRate=-0.03848825395107269}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T14:49:25,504 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731422965379Initializing all the Stores at 1731422965381 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731422965382 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422965382Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422965382Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422965383 (+1 ms)Cleaning up temporary data from old regions at 1731422965479 (+96 ms)Region opened successfully at 1731422965504 (+25 ms) 2024-11-12T14:49:25,506 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-12T14:49:25,543 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e85fc16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3945e1ca6f47/172.17.0.2:0 2024-11-12T14:49:25,575 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-12T14:49:25,585 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T14:49:25,585 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T14:49:25,588 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T14:49:25,589 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-12T14:49:25,596 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-12T14:49:25,596 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-12T14:49:25,625 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-12T14:49:25,635 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-12T14:49:25,636 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-12T14:49:25,639 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-12T14:49:25,641 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-12T14:49:25,642 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-12T14:49:25,645 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-12T14:49:25,654 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-12T14:49:25,655 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-12T14:49:25,657 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-12T14:49:25,659 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-12T14:49:25,678 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-12T14:49:25,679 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-12T14:49:25,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37185-0x1003469214f0001, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T14:49:25,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T14:49:25,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T14:49:25,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37185-0x1003469214f0001, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:49:25,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T14:49:25,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:49:25,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:49:25,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:49:25,688 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=3945e1ca6f47,38303,1731422963767, sessionid=0x1003469214f0000, setting cluster-up flag (Was=false) 2024-11-12T14:49:25,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:49:25,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:49:25,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37185-0x1003469214f0001, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:49:25,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:49:25,705 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-12T14:49:25,707 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3945e1ca6f47,38303,1731422963767 2024-11-12T14:49:25,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:49:25,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37185-0x1003469214f0001, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:49:25,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:49:25,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:49:25,717 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-12T14:49:25,718 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=3945e1ca6f47,38303,1731422963767 2024-11-12T14:49:25,725 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-12T14:49:25,800 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T14:49:25,810 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-12T14:49:25,816 INFO [RS:0;3945e1ca6f47:37185 {}] regionserver.HRegionServer(746): ClusterId : 1fc95813-043e-41c4-8f96-df8b2f6ddc4d 2024-11-12T14:49:25,816 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(746): ClusterId : 1fc95813-043e-41c4-8f96-df8b2f6ddc4d 2024-11-12T14:49:25,819 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.HRegionServer(746): ClusterId : 1fc95813-043e-41c4-8f96-df8b2f6ddc4d 2024-11-12T14:49:25,819 DEBUG [RS:1;3945e1ca6f47:45845 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T14:49:25,819 DEBUG [RS:2;3945e1ca6f47:35473 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T14:49:25,819 DEBUG [RS:0;3945e1ca6f47:37185 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T14:49:25,820 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-12T14:49:25,825 DEBUG [RS:0;3945e1ca6f47:37185 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T14:49:25,825 DEBUG [RS:1;3945e1ca6f47:45845 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T14:49:25,825 DEBUG [RS:0;3945e1ca6f47:37185 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T14:49:25,825 DEBUG [RS:1;3945e1ca6f47:45845 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T14:49:25,827 DEBUG [RS:2;3945e1ca6f47:35473 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T14:49:25,827 DEBUG [RS:2;3945e1ca6f47:35473 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T14:49:25,829 DEBUG [RS:1;3945e1ca6f47:45845 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T14:49:25,829 DEBUG [RS:0;3945e1ca6f47:37185 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T14:49:25,830 DEBUG [RS:0;3945e1ca6f47:37185 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c68e27d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3945e1ca6f47/172.17.0.2:0 2024-11-12T14:49:25,830 DEBUG [RS:1;3945e1ca6f47:45845 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4524dcde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3945e1ca6f47/172.17.0.2:0 2024-11-12T14:49:25,830 DEBUG [RS:2;3945e1ca6f47:35473 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T14:49:25,831 DEBUG [RS:2;3945e1ca6f47:35473 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15c3fb42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=3945e1ca6f47/172.17.0.2:0 2024-11-12T14:49:25,830 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 3945e1ca6f47,38303,1731422963767 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-12T14:49:25,842 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/3945e1ca6f47:0, corePoolSize=5, maxPoolSize=5 2024-11-12T14:49:25,842 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/3945e1ca6f47:0, corePoolSize=5, maxPoolSize=5 2024-11-12T14:49:25,842 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/3945e1ca6f47:0, corePoolSize=5, maxPoolSize=5 2024-11-12T14:49:25,842 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/3945e1ca6f47:0, corePoolSize=5, maxPoolSize=5 2024-11-12T14:49:25,842 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/3945e1ca6f47:0, corePoolSize=10, maxPoolSize=10 2024-11-12T14:49:25,843 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:25,843 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/3945e1ca6f47:0, corePoolSize=2, maxPoolSize=2 2024-11-12T14:49:25,843 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:25,847 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731422995847 2024-11-12T14:49:25,847 DEBUG [RS:2;3945e1ca6f47:35473 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;3945e1ca6f47:35473 2024-11-12T14:49:25,849 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-12T14:49:25,850 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T14:49:25,850 DEBUG [RS:0;3945e1ca6f47:37185 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;3945e1ca6f47:37185 2024-11-12T14:49:25,850 DEBUG [RS:1;3945e1ca6f47:45845 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;3945e1ca6f47:45845 2024-11-12T14:49:25,851 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-12T14:49:25,851 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T14:49:25,851 INFO [RS:0;3945e1ca6f47:37185 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T14:49:25,851 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T14:49:25,851 DEBUG [RS:2;3945e1ca6f47:35473 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T14:49:25,851 INFO [RS:0;3945e1ca6f47:37185 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T14:49:25,851 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T14:49:25,851 DEBUG [RS:0;3945e1ca6f47:37185 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T14:49:25,851 DEBUG [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T14:49:25,854 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(2659): reportForDuty to master=3945e1ca6f47,38303,1731422963767 with port=45845, startcode=1731422964694 2024-11-12T14:49:25,854 INFO [RS:0;3945e1ca6f47:37185 {}] regionserver.HRegionServer(2659): reportForDuty to master=3945e1ca6f47,38303,1731422963767 with port=37185, startcode=1731422964568 2024-11-12T14:49:25,855 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-12T14:49:25,856 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-12T14:49:25,856 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.HRegionServer(2659): reportForDuty to master=3945e1ca6f47,38303,1731422963767 with port=35473, startcode=1731422964757 2024-11-12T14:49:25,856 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-12T14:49:25,856 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-12T14:49:25,857 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:25,861 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T14:49:25,861 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-12T14:49:25,862 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-12T14:49:25,863 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-12T14:49:25,863 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-12T14:49:25,866 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-12T14:49:25,866 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-12T14:49:25,867 DEBUG [RS:0;3945e1ca6f47:37185 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T14:49:25,867 DEBUG [RS:2;3945e1ca6f47:35473 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T14:49:25,867 DEBUG [RS:1;3945e1ca6f47:45845 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T14:49:25,868 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:25,868 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-12T14:49:25,870 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/3945e1ca6f47:0:becomeActiveMaster-HFileCleaner.large.0-1731422965867,5,FailOnTimeoutGroup] 2024-11-12T14:49:25,878 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/3945e1ca6f47:0:becomeActiveMaster-HFileCleaner.small.0-1731422965871,5,FailOnTimeoutGroup] 2024-11-12T14:49:25,879 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:25,879 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-12T14:49:25,880 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:25,881 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:25,956 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48171, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T14:49:25,956 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57439, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T14:49:25,956 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55999, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T14:49:25,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741831_1007 (size=1321) 2024-11-12T14:49:25,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741831_1007 (size=1321) 2024-11-12T14:49:25,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741831_1007 (size=1321) 2024-11-12T14:49:25,964 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38303 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3945e1ca6f47,35473,1731422964757 2024-11-12T14:49:25,965 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-12T14:49:25,966 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 2024-11-12T14:49:25,967 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38303 {}] master.ServerManager(517): Registering regionserver=3945e1ca6f47,35473,1731422964757 2024-11-12T14:49:25,978 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38303 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3945e1ca6f47,37185,1731422964568 2024-11-12T14:49:25,978 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38303 {}] master.ServerManager(517): Registering regionserver=3945e1ca6f47,37185,1731422964568 2024-11-12T14:49:25,982 DEBUG [RS:2;3945e1ca6f47:35473 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 2024-11-12T14:49:25,983 DEBUG [RS:2;3945e1ca6f47:35473 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37295 2024-11-12T14:49:25,983 DEBUG [RS:2;3945e1ca6f47:35473 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T14:49:25,984 DEBUG [RS:0;3945e1ca6f47:37185 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 2024-11-12T14:49:25,984 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38303 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 3945e1ca6f47,45845,1731422964694 2024-11-12T14:49:25,984 DEBUG [RS:0;3945e1ca6f47:37185 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37295 2024-11-12T14:49:25,984 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38303 {}] master.ServerManager(517): Registering regionserver=3945e1ca6f47,45845,1731422964694 2024-11-12T14:49:25,984 DEBUG [RS:0;3945e1ca6f47:37185 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T14:49:25,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T14:49:25,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741832_1008 (size=32) 2024-11-12T14:49:25,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741832_1008 (size=32) 2024-11-12T14:49:25,989 DEBUG [RS:2;3945e1ca6f47:35473 {}] zookeeper.ZKUtil(111): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3945e1ca6f47,35473,1731422964757 2024-11-12T14:49:25,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741832_1008 (size=32) 2024-11-12T14:49:25,990 DEBUG [RS:0;3945e1ca6f47:37185 {}] zookeeper.ZKUtil(111): regionserver:37185-0x1003469214f0001, quorum=127.0.0.1:62355, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3945e1ca6f47,37185,1731422964568 2024-11-12T14:49:25,990 WARN [RS:2;3945e1ca6f47:35473 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T14:49:25,990 WARN [RS:0;3945e1ca6f47:37185 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T14:49:25,990 INFO [RS:2;3945e1ca6f47:35473 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:49:25,990 INFO [RS:0;3945e1ca6f47:37185 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:49:25,990 DEBUG [RS:0;3945e1ca6f47:37185 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568 2024-11-12T14:49:25,990 DEBUG [RS:2;3945e1ca6f47:35473 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,35473,1731422964757 2024-11-12T14:49:25,990 DEBUG [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 2024-11-12T14:49:25,990 DEBUG [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37295 2024-11-12T14:49:25,990 DEBUG [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T14:49:25,992 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:25,993 DEBUG [RS:1;3945e1ca6f47:45845 {}] zookeeper.ZKUtil(111): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/3945e1ca6f47,45845,1731422964694 2024-11-12T14:49:25,993 WARN [RS:1;3945e1ca6f47:45845 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T14:49:25,993 INFO [RS:1;3945e1ca6f47:45845 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:49:25,994 DEBUG [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,45845,1731422964694 2024-11-12T14:49:25,994 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3945e1ca6f47,35473,1731422964757] 2024-11-12T14:49:25,994 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3945e1ca6f47,37185,1731422964568] 2024-11-12T14:49:25,995 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [3945e1ca6f47,45845,1731422964694] 2024-11-12T14:49:25,997 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T14:49:26,002 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T14:49:26,003 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:26,004 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:49:26,004 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T14:49:26,008 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T14:49:26,009 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:26,010 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:49:26,010 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T14:49:26,015 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T14:49:26,015 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:26,017 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:49:26,017 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T14:49:26,021 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T14:49:26,021 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:26,023 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:49:26,023 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T14:49:26,025 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740 2024-11-12T14:49:26,026 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740 2024-11-12T14:49:26,028 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T14:49:26,029 INFO [RS:0;3945e1ca6f47:37185 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T14:49:26,030 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T14:49:26,030 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T14:49:26,031 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T14:49:26,032 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T14:49:26,035 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T14:49:26,042 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:49:26,044 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63685146, jitterRate=-0.05101737380027771}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T14:49:26,050 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731422965992Initializing all the Stores at 1731422965994 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731422965994Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731422965997 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422965997Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731422965997Cleaning up temporary data from old regions at 1731422966031 (+34 ms)Region opened successfully at 1731422966049 (+18 ms) 2024-11-12T14:49:26,050 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T14:49:26,050 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T14:49:26,050 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T14:49:26,050 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T14:49:26,050 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T14:49:26,052 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T14:49:26,052 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731422966050Disabling compacts and flushes for region at 1731422966050Disabling writes for close at 1731422966050Writing region close event to WAL at 1731422966051 (+1 ms)Closed at 1731422966052 (+1 ms) 2024-11-12T14:49:26,054 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T14:49:26,056 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T14:49:26,056 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-12T14:49:26,063 INFO [RS:1;3945e1ca6f47:45845 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T14:49:26,063 INFO [RS:1;3945e1ca6f47:45845 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:26,065 INFO [RS:0;3945e1ca6f47:37185 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T14:49:26,065 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T14:49:26,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T14:49:26,070 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T14:49:26,077 INFO [RS:2;3945e1ca6f47:35473 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T14:49:26,077 INFO [RS:0;3945e1ca6f47:37185 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T14:49:26,077 INFO [RS:2;3945e1ca6f47:35473 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:26,077 INFO [RS:0;3945e1ca6f47:37185 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:26,079 INFO [RS:1;3945e1ca6f47:45845 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T14:49:26,081 INFO [RS:1;3945e1ca6f47:45845 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:26,081 DEBUG [RS:1;3945e1ca6f47:45845 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,082 DEBUG [RS:1;3945e1ca6f47:45845 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,082 DEBUG [RS:1;3945e1ca6f47:45845 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,082 DEBUG [RS:1;3945e1ca6f47:45845 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,082 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T14:49:26,082 DEBUG [RS:1;3945e1ca6f47:45845 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,083 DEBUG [RS:1;3945e1ca6f47:45845 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0, corePoolSize=2, maxPoolSize=2 2024-11-12T14:49:26,083 DEBUG [RS:1;3945e1ca6f47:45845 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,083 DEBUG [RS:1;3945e1ca6f47:45845 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,083 DEBUG [RS:1;3945e1ca6f47:45845 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,083 DEBUG [RS:1;3945e1ca6f47:45845 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,084 DEBUG [RS:1;3945e1ca6f47:45845 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,084 DEBUG [RS:1;3945e1ca6f47:45845 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,084 DEBUG [RS:1;3945e1ca6f47:45845 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3945e1ca6f47:0, corePoolSize=3, maxPoolSize=3 2024-11-12T14:49:26,084 DEBUG [RS:1;3945e1ca6f47:45845 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3945e1ca6f47:0, corePoolSize=3, maxPoolSize=3 2024-11-12T14:49:26,084 INFO [RS:0;3945e1ca6f47:37185 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T14:49:26,086 INFO [RS:0;3945e1ca6f47:37185 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T14:49:26,087 INFO [RS:0;3945e1ca6f47:37185 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:26,087 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-12T14:49:26,087 DEBUG [RS:0;3945e1ca6f47:37185 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,087 DEBUG [RS:0;3945e1ca6f47:37185 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,088 DEBUG [RS:0;3945e1ca6f47:37185 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,088 DEBUG [RS:0;3945e1ca6f47:37185 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,088 DEBUG [RS:0;3945e1ca6f47:37185 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,088 DEBUG [RS:0;3945e1ca6f47:37185 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0, corePoolSize=2, maxPoolSize=2 2024-11-12T14:49:26,089 DEBUG [RS:0;3945e1ca6f47:37185 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,089 DEBUG [RS:0;3945e1ca6f47:37185 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,089 DEBUG [RS:0;3945e1ca6f47:37185 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,089 DEBUG [RS:0;3945e1ca6f47:37185 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,089 DEBUG [RS:0;3945e1ca6f47:37185 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,089 DEBUG [RS:0;3945e1ca6f47:37185 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,090 DEBUG [RS:0;3945e1ca6f47:37185 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3945e1ca6f47:0, corePoolSize=3, maxPoolSize=3 2024-11-12T14:49:26,090 DEBUG [RS:0;3945e1ca6f47:37185 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3945e1ca6f47:0, corePoolSize=3, maxPoolSize=3 2024-11-12T14:49:26,095 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T14:49:26,097 INFO [RS:2;3945e1ca6f47:35473 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T14:49:26,098 INFO [RS:2;3945e1ca6f47:35473 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:26,098 DEBUG [RS:2;3945e1ca6f47:35473 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,098 DEBUG [RS:2;3945e1ca6f47:35473 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,098 DEBUG [RS:2;3945e1ca6f47:35473 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,099 DEBUG [RS:2;3945e1ca6f47:35473 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,099 DEBUG [RS:2;3945e1ca6f47:35473 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,099 DEBUG [RS:2;3945e1ca6f47:35473 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0, corePoolSize=2, maxPoolSize=2 2024-11-12T14:49:26,099 DEBUG [RS:2;3945e1ca6f47:35473 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,099 DEBUG [RS:2;3945e1ca6f47:35473 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,100 DEBUG [RS:2;3945e1ca6f47:35473 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,100 DEBUG [RS:2;3945e1ca6f47:35473 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,100 DEBUG [RS:2;3945e1ca6f47:35473 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,100 DEBUG [RS:2;3945e1ca6f47:35473 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/3945e1ca6f47:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:49:26,100 DEBUG [RS:2;3945e1ca6f47:35473 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/3945e1ca6f47:0, corePoolSize=3, maxPoolSize=3 2024-11-12T14:49:26,100 DEBUG [RS:2;3945e1ca6f47:35473 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/3945e1ca6f47:0, corePoolSize=3, maxPoolSize=3 2024-11-12T14:49:26,115 INFO [RS:1;3945e1ca6f47:45845 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:26,115 INFO [RS:1;3945e1ca6f47:45845 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:26,115 INFO [RS:1;3945e1ca6f47:45845 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:26,115 INFO [RS:1;3945e1ca6f47:45845 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:26,116 INFO [RS:1;3945e1ca6f47:45845 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:26,116 INFO [RS:1;3945e1ca6f47:45845 {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,45845,1731422964694-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T14:49:26,117 INFO [RS:0;3945e1ca6f47:37185 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:26,117 INFO [RS:0;3945e1ca6f47:37185 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:26,117 INFO [RS:0;3945e1ca6f47:37185 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:26,117 INFO [RS:0;3945e1ca6f47:37185 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:26,117 INFO [RS:0;3945e1ca6f47:37185 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:26,118 INFO [RS:0;3945e1ca6f47:37185 {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,37185,1731422964568-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T14:49:26,122 INFO [RS:2;3945e1ca6f47:35473 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:26,123 INFO [RS:2;3945e1ca6f47:35473 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:26,123 INFO [RS:2;3945e1ca6f47:35473 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:26,123 INFO [RS:2;3945e1ca6f47:35473 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:26,123 INFO [RS:2;3945e1ca6f47:35473 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:26,123 INFO [RS:2;3945e1ca6f47:35473 {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,35473,1731422964757-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T14:49:26,155 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T14:49:26,156 INFO [RS:0;3945e1ca6f47:37185 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T14:49:26,158 INFO [RS:0;3945e1ca6f47:37185 {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,37185,1731422964568-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:26,159 INFO [RS:1;3945e1ca6f47:45845 {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,45845,1731422964694-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:26,159 INFO [RS:1;3945e1ca6f47:45845 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:26,159 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.Replication(171): 3945e1ca6f47,45845,1731422964694 started 2024-11-12T14:49:26,167 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T14:49:26,168 INFO [RS:2;3945e1ca6f47:35473 {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,35473,1731422964757-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:26,168 INFO [RS:0;3945e1ca6f47:37185 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:26,168 INFO [RS:0;3945e1ca6f47:37185 {}] regionserver.Replication(171): 3945e1ca6f47,37185,1731422964568 started 2024-11-12T14:49:26,169 INFO [RS:2;3945e1ca6f47:35473 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:26,169 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.Replication(171): 3945e1ca6f47,35473,1731422964757 started 2024-11-12T14:49:26,195 INFO [RS:2;3945e1ca6f47:35473 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:26,196 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.HRegionServer(1482): Serving as 3945e1ca6f47,35473,1731422964757, RpcServer on 3945e1ca6f47/172.17.0.2:35473, sessionid=0x1003469214f0003 2024-11-12T14:49:26,197 DEBUG [RS:2;3945e1ca6f47:35473 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T14:49:26,197 DEBUG [RS:2;3945e1ca6f47:35473 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3945e1ca6f47,35473,1731422964757 2024-11-12T14:49:26,197 DEBUG [RS:2;3945e1ca6f47:35473 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3945e1ca6f47,35473,1731422964757' 2024-11-12T14:49:26,197 DEBUG [RS:2;3945e1ca6f47:35473 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T14:49:26,199 DEBUG [RS:2;3945e1ca6f47:35473 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T14:49:26,201 DEBUG [RS:2;3945e1ca6f47:35473 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T14:49:26,201 DEBUG [RS:2;3945e1ca6f47:35473 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T14:49:26,201 DEBUG [RS:2;3945e1ca6f47:35473 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3945e1ca6f47,35473,1731422964757 2024-11-12T14:49:26,201 DEBUG [RS:2;3945e1ca6f47:35473 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3945e1ca6f47,35473,1731422964757' 2024-11-12T14:49:26,201 DEBUG [RS:2;3945e1ca6f47:35473 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T14:49:26,202 DEBUG [RS:2;3945e1ca6f47:35473 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T14:49:26,202 INFO [RS:1;3945e1ca6f47:45845 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:26,202 INFO [RS:0;3945e1ca6f47:37185 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:26,203 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(1482): Serving as 3945e1ca6f47,45845,1731422964694, RpcServer on 3945e1ca6f47/172.17.0.2:45845, sessionid=0x1003469214f0002 2024-11-12T14:49:26,203 DEBUG [RS:1;3945e1ca6f47:45845 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T14:49:26,203 DEBUG [RS:1;3945e1ca6f47:45845 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3945e1ca6f47,45845,1731422964694 2024-11-12T14:49:26,203 DEBUG [RS:2;3945e1ca6f47:35473 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T14:49:26,203 DEBUG [RS:1;3945e1ca6f47:45845 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3945e1ca6f47,45845,1731422964694' 2024-11-12T14:49:26,203 DEBUG [RS:1;3945e1ca6f47:45845 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T14:49:26,203 INFO [RS:2;3945e1ca6f47:35473 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T14:49:26,203 INFO [RS:2;3945e1ca6f47:35473 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T14:49:26,203 INFO [RS:0;3945e1ca6f47:37185 {}] regionserver.HRegionServer(1482): Serving as 3945e1ca6f47,37185,1731422964568, RpcServer on 3945e1ca6f47/172.17.0.2:37185, sessionid=0x1003469214f0001 2024-11-12T14:49:26,204 DEBUG [RS:0;3945e1ca6f47:37185 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T14:49:26,204 DEBUG [RS:0;3945e1ca6f47:37185 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 3945e1ca6f47,37185,1731422964568 2024-11-12T14:49:26,204 DEBUG [RS:0;3945e1ca6f47:37185 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3945e1ca6f47,37185,1731422964568' 2024-11-12T14:49:26,204 DEBUG [RS:0;3945e1ca6f47:37185 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T14:49:26,204 DEBUG [RS:1;3945e1ca6f47:45845 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T14:49:26,205 DEBUG [RS:0;3945e1ca6f47:37185 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T14:49:26,205 DEBUG [RS:1;3945e1ca6f47:45845 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T14:49:26,205 DEBUG [RS:1;3945e1ca6f47:45845 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T14:49:26,205 DEBUG [RS:1;3945e1ca6f47:45845 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3945e1ca6f47,45845,1731422964694 2024-11-12T14:49:26,205 DEBUG [RS:1;3945e1ca6f47:45845 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3945e1ca6f47,45845,1731422964694' 2024-11-12T14:49:26,205 DEBUG [RS:1;3945e1ca6f47:45845 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T14:49:26,205 DEBUG [RS:0;3945e1ca6f47:37185 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T14:49:26,206 DEBUG [RS:1;3945e1ca6f47:45845 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T14:49:26,207 DEBUG [RS:1;3945e1ca6f47:45845 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T14:49:26,207 INFO [RS:1;3945e1ca6f47:45845 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T14:49:26,207 INFO [RS:1;3945e1ca6f47:45845 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T14:49:26,209 DEBUG [RS:0;3945e1ca6f47:37185 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T14:49:26,209 DEBUG [RS:0;3945e1ca6f47:37185 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 3945e1ca6f47,37185,1731422964568 2024-11-12T14:49:26,210 DEBUG [RS:0;3945e1ca6f47:37185 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '3945e1ca6f47,37185,1731422964568' 2024-11-12T14:49:26,210 DEBUG [RS:0;3945e1ca6f47:37185 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T14:49:26,211 DEBUG [RS:0;3945e1ca6f47:37185 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T14:49:26,212 DEBUG [RS:0;3945e1ca6f47:37185 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T14:49:26,212 INFO [RS:0;3945e1ca6f47:37185 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T14:49:26,212 INFO [RS:0;3945e1ca6f47:37185 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T14:49:26,238 WARN [3945e1ca6f47:38303 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-12T14:49:26,309 INFO [RS:1;3945e1ca6f47:45845 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T14:49:26,309 INFO [RS:2;3945e1ca6f47:35473 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T14:49:26,312 INFO [RS:1;3945e1ca6f47:45845 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3945e1ca6f47%2C45845%2C1731422964694, suffix=, logDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,45845,1731422964694, archiveDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/oldWALs, maxLogs=32 2024-11-12T14:49:26,313 INFO [RS:0;3945e1ca6f47:37185 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T14:49:26,315 INFO [RS:2;3945e1ca6f47:35473 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3945e1ca6f47%2C35473%2C1731422964757, suffix=, logDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,35473,1731422964757, archiveDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/oldWALs, maxLogs=32 2024-11-12T14:49:26,318 INFO [RS:0;3945e1ca6f47:37185 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3945e1ca6f47%2C37185%2C1731422964568, suffix=, logDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568, archiveDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/oldWALs, maxLogs=32 2024-11-12T14:49:26,335 DEBUG [RS:2;3945e1ca6f47:35473 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,35473,1731422964757/3945e1ca6f47%2C35473%2C1731422964757.1731422966318, exclude list is [], retry=0 2024-11-12T14:49:26,335 DEBUG [RS:0;3945e1ca6f47:37185 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568/3945e1ca6f47%2C37185%2C1731422964568.1731422966319, exclude list is [], retry=0 2024-11-12T14:49:26,342 DEBUG [RS:1;3945e1ca6f47:45845 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,45845,1731422964694/3945e1ca6f47%2C45845%2C1731422964694.1731422966317, exclude list is [], retry=0 2024-11-12T14:49:26,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38947,DS-2fd665aa-69eb-44fd-bca0-32d25d7b0e8b,DISK] 2024-11-12T14:49:26,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41429,DS-7b068bce-ec51-4d06-8fdd-88c861866bd0,DISK] 2024-11-12T14:49:26,345 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44401,DS-48b2b78b-5966-4c15-86cf-f894eccf2d5b,DISK] 2024-11-12T14:49:26,348 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38947,DS-2fd665aa-69eb-44fd-bca0-32d25d7b0e8b,DISK] 2024-11-12T14:49:26,354 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44401,DS-48b2b78b-5966-4c15-86cf-f894eccf2d5b,DISK] 2024-11-12T14:49:26,355 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38947,DS-2fd665aa-69eb-44fd-bca0-32d25d7b0e8b,DISK] 2024-11-12T14:49:26,355 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41429,DS-7b068bce-ec51-4d06-8fdd-88c861866bd0,DISK] 2024-11-12T14:49:26,355 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41429,DS-7b068bce-ec51-4d06-8fdd-88c861866bd0,DISK] 2024-11-12T14:49:26,356 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44401,DS-48b2b78b-5966-4c15-86cf-f894eccf2d5b,DISK] 2024-11-12T14:49:26,369 INFO [RS:0;3945e1ca6f47:37185 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568/3945e1ca6f47%2C37185%2C1731422964568.1731422966319 2024-11-12T14:49:26,379 DEBUG [RS:0;3945e1ca6f47:37185 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36551:36551),(127.0.0.1/127.0.0.1:38447:38447),(127.0.0.1/127.0.0.1:34963:34963)] 2024-11-12T14:49:26,386 INFO [RS:1;3945e1ca6f47:45845 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,45845,1731422964694/3945e1ca6f47%2C45845%2C1731422964694.1731422966317 2024-11-12T14:49:26,389 INFO [RS:2;3945e1ca6f47:35473 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,35473,1731422964757/3945e1ca6f47%2C35473%2C1731422964757.1731422966318 2024-11-12T14:49:26,389 DEBUG [RS:1;3945e1ca6f47:45845 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34963:34963),(127.0.0.1/127.0.0.1:36551:36551),(127.0.0.1/127.0.0.1:38447:38447)] 2024-11-12T14:49:26,390 DEBUG [RS:2;3945e1ca6f47:35473 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34963:34963),(127.0.0.1/127.0.0.1:38447:38447),(127.0.0.1/127.0.0.1:36551:36551)] 2024-11-12T14:49:26,490 DEBUG [3945e1ca6f47:38303 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-12T14:49:26,497 DEBUG [3945e1ca6f47:38303 {}] balancer.BalancerClusterState(204): Hosts are {3945e1ca6f47=0} racks are {/default-rack=0} 2024-11-12T14:49:26,504 DEBUG [3945e1ca6f47:38303 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T14:49:26,504 DEBUG [3945e1ca6f47:38303 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T14:49:26,504 DEBUG [3945e1ca6f47:38303 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-12T14:49:26,504 DEBUG [3945e1ca6f47:38303 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T14:49:26,504 DEBUG [3945e1ca6f47:38303 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T14:49:26,504 DEBUG [3945e1ca6f47:38303 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-12T14:49:26,504 INFO [3945e1ca6f47:38303 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T14:49:26,504 INFO [3945e1ca6f47:38303 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T14:49:26,504 INFO [3945e1ca6f47:38303 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-12T14:49:26,504 DEBUG [3945e1ca6f47:38303 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T14:49:26,512 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=3945e1ca6f47,37185,1731422964568 2024-11-12T14:49:26,521 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3945e1ca6f47,37185,1731422964568, state=OPENING 2024-11-12T14:49:26,529 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-12T14:49:26,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37185-0x1003469214f0001, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:49:26,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:49:26,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:49:26,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:49:26,532 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:49:26,532 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:49:26,532 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:49:26,532 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:49:26,534 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T14:49:26,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=3945e1ca6f47,37185,1731422964568}] 2024-11-12T14:49:26,714 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T14:49:26,716 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58691, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T14:49:26,730 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-12T14:49:26,731 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:49:26,731 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-12T14:49:26,735 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3945e1ca6f47%2C37185%2C1731422964568.meta, suffix=.meta, logDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568, archiveDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/oldWALs, maxLogs=32 2024-11-12T14:49:26,750 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568/3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, exclude list is [], retry=0 2024-11-12T14:49:26,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44401,DS-48b2b78b-5966-4c15-86cf-f894eccf2d5b,DISK] 2024-11-12T14:49:26,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38947,DS-2fd665aa-69eb-44fd-bca0-32d25d7b0e8b,DISK] 2024-11-12T14:49:26,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41429,DS-7b068bce-ec51-4d06-8fdd-88c861866bd0,DISK] 2024-11-12T14:49:26,761 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568/3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta 2024-11-12T14:49:26,765 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38447:38447),(127.0.0.1/127.0.0.1:34963:34963),(127.0.0.1/127.0.0.1:36551:36551)] 2024-11-12T14:49:26,766 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:49:26,767 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-12T14:49:26,770 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-12T14:49:26,775 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-12T14:49:26,780 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-12T14:49:26,781 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:26,781 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-12T14:49:26,781 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-12T14:49:26,784 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T14:49:26,786 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T14:49:26,786 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:26,787 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:49:26,788 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T14:49:26,790 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T14:49:26,790 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:26,791 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:49:26,791 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T14:49:26,793 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T14:49:26,793 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:26,795 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:49:26,795 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T14:49:26,796 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T14:49:26,796 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:26,797 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:49:26,798 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T14:49:26,799 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740 2024-11-12T14:49:26,803 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740 2024-11-12T14:49:26,806 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T14:49:26,806 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T14:49:26,807 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T14:49:26,811 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T14:49:26,814 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59881765, jitterRate=-0.1076921671628952}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T14:49:26,814 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-12T14:49:26,816 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731422966781Writing region info on filesystem at 1731422966782 (+1 ms)Initializing all the Stores at 1731422966784 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731422966784Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731422966784Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422966784Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731422966784Cleaning up temporary data from old regions at 1731422966806 (+22 ms)Running coprocessor post-open hooks at 1731422966814 (+8 ms)Region opened successfully at 1731422966816 (+2 ms) 2024-11-12T14:49:26,824 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731422966704 2024-11-12T14:49:26,836 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-12T14:49:26,837 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-12T14:49:26,840 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,37185,1731422964568 2024-11-12T14:49:26,843 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3945e1ca6f47,37185,1731422964568, state=OPEN 2024-11-12T14:49:26,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37185-0x1003469214f0001, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:49:26,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:49:26,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:49:26,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:49:26,845 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:49:26,845 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:49:26,845 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:49:26,845 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:49:26,846 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=3945e1ca6f47,37185,1731422964568 2024-11-12T14:49:26,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-12T14:49:26,854 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3945e1ca6f47,37185,1731422964568 in 310 msec 2024-11-12T14:49:26,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-12T14:49:26,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 791 msec 2024-11-12T14:49:26,867 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T14:49:26,867 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-12T14:49:26,891 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:49:26,892 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:49:26,925 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:49:26,927 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47519, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:49:26,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1920 sec 2024-11-12T14:49:26,950 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731422966950, completionTime=-1 2024-11-12T14:49:26,986 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-12T14:49:26,986 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T14:49:27,026 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-12T14:49:27,027 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731423027026 2024-11-12T14:49:27,027 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731423087027 2024-11-12T14:49:27,027 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 40 msec 2024-11-12T14:49:27,031 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-12T14:49:27,040 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,38303,1731422963767-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:27,040 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,38303,1731422963767-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:27,040 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,38303,1731422963767-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:27,042 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-3945e1ca6f47:38303, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:27,042 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:27,045 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:27,050 DEBUG [master/3945e1ca6f47:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-12T14:49:27,079 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.194sec 2024-11-12T14:49:27,082 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-12T14:49:27,083 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-12T14:49:27,085 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-12T14:49:27,085 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-12T14:49:27,085 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-12T14:49:27,086 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,38303,1731422963767-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T14:49:27,087 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,38303,1731422963767-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-12T14:49:27,094 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-12T14:49:27,095 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-12T14:49:27,096 INFO [master/3945e1ca6f47:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=3945e1ca6f47,38303,1731422963767-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:49:27,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61693573, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:49:27,139 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-12T14:49:27,140 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-12T14:49:27,145 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 3945e1ca6f47,38303,-1 for getting cluster id 2024-11-12T14:49:27,148 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:49:27,158 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1fc95813-043e-41c4-8f96-df8b2f6ddc4d' 2024-11-12T14:49:27,161 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:49:27,161 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1fc95813-043e-41c4-8f96-df8b2f6ddc4d" 2024-11-12T14:49:27,163 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@398c4b4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:49:27,164 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [3945e1ca6f47,38303,-1] 2024-11-12T14:49:27,166 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:49:27,169 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:49:27,171 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:49:27,174 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5499896f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:49:27,174 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:49:27,183 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:49:27,184 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:49:27,187 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:49:27,211 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=3945e1ca6f47,38303,1731422963767 2024-11-12T14:49:27,212 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-12T14:49:27,218 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T14:49:27,222 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 3945e1ca6f47,38303,1731422963767 2024-11-12T14:49:27,224 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@48f67e32 2024-11-12T14:49:27,225 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T14:49:27,227 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T14:49:27,236 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38303 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T14:49:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38303 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-12T14:49:27,248 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T14:49:27,250 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38303 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true" procId is: 4 2024-11-12T14:49:27,250 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:27,255 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T14:49:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T14:49:27,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741837_1013 (size=1020) 2024-11-12T14:49:27,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741837_1013 (size=1020) 2024-11-12T14:49:27,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741837_1013 (size=1020) 2024-11-12T14:49:27,314 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f943ce043844f379c1b7ecbf76b5a419, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 2024-11-12T14:49:27,314 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 06aa3b2e56ae4404b602d6fa98eabe33, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 2024-11-12T14:49:27,315 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 489408be5f35a96bee1ee0f68f064599, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731422967229.489408be5f35a96bee1ee0f68f064599.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 2024-11-12T14:49:27,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741838_1014 (size=110) 2024-11-12T14:49:27,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741838_1014 (size=110) 2024-11-12T14:49:27,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741838_1014 (size=110) 2024-11-12T14:49:27,361 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:27,362 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 06aa3b2e56ae4404b602d6fa98eabe33, disabling compactions & flushes 2024-11-12T14:49:27,362 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33. 2024-11-12T14:49:27,362 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33. 2024-11-12T14:49:27,362 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33. after waiting 0 ms 2024-11-12T14:49:27,362 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33. 2024-11-12T14:49:27,362 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33. 2024-11-12T14:49:27,362 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 06aa3b2e56ae4404b602d6fa98eabe33: Waiting for close lock at 1731422967362Disabling compacts and flushes for region at 1731422967362Disabling writes for close at 1731422967362Writing region close event to WAL at 1731422967362Closed at 1731422967362 2024-11-12T14:49:27,363 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => cf0006d9b8777107f458b7008fc42ba9, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731422967229.cf0006d9b8777107f458b7008fc42ba9.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 2024-11-12T14:49:27,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T14:49:27,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741839_1015 (size=107) 2024-11-12T14:49:27,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741839_1015 (size=107) 2024-11-12T14:49:27,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741839_1015 (size=107) 2024-11-12T14:49:27,373 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:27,374 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing f943ce043844f379c1b7ecbf76b5a419, disabling compactions & flushes 2024-11-12T14:49:27,374 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419. 2024-11-12T14:49:27,374 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419. 2024-11-12T14:49:27,374 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419. after waiting 0 ms 2024-11-12T14:49:27,374 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419. 2024-11-12T14:49:27,374 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419. 2024-11-12T14:49:27,374 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for f943ce043844f379c1b7ecbf76b5a419: Waiting for close lock at 1731422967374Disabling compacts and flushes for region at 1731422967374Disabling writes for close at 1731422967374Writing region close event to WAL at 1731422967374Closed at 1731422967374 2024-11-12T14:49:27,374 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d3c298441d993e84c2d9e579914483db, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 2024-11-12T14:49:27,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741840_1016 (size=110) 2024-11-12T14:49:27,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741840_1016 (size=110) 2024-11-12T14:49:27,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741840_1016 (size=110) 2024-11-12T14:49:27,385 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731422967229.489408be5f35a96bee1ee0f68f064599.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:27,385 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 489408be5f35a96bee1ee0f68f064599, disabling compactions & flushes 2024-11-12T14:49:27,385 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731422967229.489408be5f35a96bee1ee0f68f064599. 2024-11-12T14:49:27,385 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731422967229.489408be5f35a96bee1ee0f68f064599. 2024-11-12T14:49:27,385 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731422967229.489408be5f35a96bee1ee0f68f064599. after waiting 0 ms 2024-11-12T14:49:27,385 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731422967229.489408be5f35a96bee1ee0f68f064599. 2024-11-12T14:49:27,385 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731422967229.489408be5f35a96bee1ee0f68f064599. 2024-11-12T14:49:27,386 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 489408be5f35a96bee1ee0f68f064599: Waiting for close lock at 1731422967385Disabling compacts and flushes for region at 1731422967385Disabling writes for close at 1731422967385Writing region close event to WAL at 1731422967385Closed at 1731422967385 2024-11-12T14:49:27,387 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 59fa7126e2ade1f9daaaeb8bd38b09e4, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731422967229.59fa7126e2ade1f9daaaeb8bd38b09e4.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 2024-11-12T14:49:27,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741841_1017 (size=110) 2024-11-12T14:49:27,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741841_1017 (size=110) 2024-11-12T14:49:27,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741841_1017 (size=110) 2024-11-12T14:49:27,451 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:27,451 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing d3c298441d993e84c2d9e579914483db, disabling compactions & flushes 2024-11-12T14:49:27,451 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db. 2024-11-12T14:49:27,451 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db. 2024-11-12T14:49:27,451 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db. after waiting 0 ms 2024-11-12T14:49:27,451 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db. 2024-11-12T14:49:27,451 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db. 2024-11-12T14:49:27,451 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for d3c298441d993e84c2d9e579914483db: Waiting for close lock at 1731422967451Disabling compacts and flushes for region at 1731422967451Disabling writes for close at 1731422967451Writing region close event to WAL at 1731422967451Closed at 1731422967451 2024-11-12T14:49:27,452 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4936dfde0b7af7ea99d36c9e0c24fefc, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 2024-11-12T14:49:27,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741842_1018 (size=110) 2024-11-12T14:49:27,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741842_1018 (size=110) 2024-11-12T14:49:27,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741842_1018 (size=110) 2024-11-12T14:49:27,470 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731422967229.cf0006d9b8777107f458b7008fc42ba9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:27,470 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing cf0006d9b8777107f458b7008fc42ba9, disabling compactions & flushes 2024-11-12T14:49:27,470 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731422967229.cf0006d9b8777107f458b7008fc42ba9. 2024-11-12T14:49:27,470 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731422967229.cf0006d9b8777107f458b7008fc42ba9. 2024-11-12T14:49:27,470 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731422967229.cf0006d9b8777107f458b7008fc42ba9. after waiting 0 ms 2024-11-12T14:49:27,470 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731422967229.cf0006d9b8777107f458b7008fc42ba9. 2024-11-12T14:49:27,470 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731422967229.cf0006d9b8777107f458b7008fc42ba9. 2024-11-12T14:49:27,470 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for cf0006d9b8777107f458b7008fc42ba9: Waiting for close lock at 1731422967470Disabling compacts and flushes for region at 1731422967470Disabling writes for close at 1731422967470Writing region close event to WAL at 1731422967470Closed at 1731422967470 2024-11-12T14:49:27,471 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => f65e71cd750425cce9713e30a75c6002, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731422967229.f65e71cd750425cce9713e30a75c6002.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 2024-11-12T14:49:27,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741843_1019 (size=110) 2024-11-12T14:49:27,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741843_1019 (size=110) 2024-11-12T14:49:27,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741843_1019 (size=110) 2024-11-12T14:49:27,477 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731422967229.59fa7126e2ade1f9daaaeb8bd38b09e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:27,477 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing 59fa7126e2ade1f9daaaeb8bd38b09e4, disabling compactions & flushes 2024-11-12T14:49:27,477 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731422967229.59fa7126e2ade1f9daaaeb8bd38b09e4. 2024-11-12T14:49:27,477 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731422967229.59fa7126e2ade1f9daaaeb8bd38b09e4. 2024-11-12T14:49:27,477 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731422967229.59fa7126e2ade1f9daaaeb8bd38b09e4. after waiting 0 ms 2024-11-12T14:49:27,477 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731422967229.59fa7126e2ade1f9daaaeb8bd38b09e4. 2024-11-12T14:49:27,477 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731422967229.59fa7126e2ade1f9daaaeb8bd38b09e4. 2024-11-12T14:49:27,478 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for 59fa7126e2ade1f9daaaeb8bd38b09e4: Waiting for close lock at 1731422967477Disabling compacts and flushes for region at 1731422967477Disabling writes for close at 1731422967477Writing region close event to WAL at 1731422967477Closed at 1731422967477 2024-11-12T14:49:27,478 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => df409d6fa4df5d8fd91534d2d90eda60, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731422967229.df409d6fa4df5d8fd91534d2d90eda60.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 2024-11-12T14:49:27,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741844_1020 (size=110) 2024-11-12T14:49:27,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741844_1020 (size=110) 2024-11-12T14:49:27,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741844_1020 (size=110) 2024-11-12T14:49:27,518 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:27,518 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 4936dfde0b7af7ea99d36c9e0c24fefc, disabling compactions & flushes 2024-11-12T14:49:27,518 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc. 2024-11-12T14:49:27,518 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc. 2024-11-12T14:49:27,518 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc. after waiting 0 ms 2024-11-12T14:49:27,519 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc. 2024-11-12T14:49:27,519 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc. 2024-11-12T14:49:27,519 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4936dfde0b7af7ea99d36c9e0c24fefc: Waiting for close lock at 1731422967518Disabling compacts and flushes for region at 1731422967518Disabling writes for close at 1731422967519 (+1 ms)Writing region close event to WAL at 1731422967519Closed at 1731422967519 2024-11-12T14:49:27,520 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 92f3e9baa717d57147319661f6f00e06, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 2024-11-12T14:49:27,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741845_1021 (size=110) 2024-11-12T14:49:27,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741845_1021 (size=110) 2024-11-12T14:49:27,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741845_1021 (size=110) 2024-11-12T14:49:27,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741846_1022 (size=110) 2024-11-12T14:49:27,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741846_1022 (size=110) 2024-11-12T14:49:27,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741846_1022 (size=110) 2024-11-12T14:49:27,541 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731422967229.df409d6fa4df5d8fd91534d2d90eda60.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:27,542 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing df409d6fa4df5d8fd91534d2d90eda60, disabling compactions & flushes 2024-11-12T14:49:27,542 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731422967229.df409d6fa4df5d8fd91534d2d90eda60. 2024-11-12T14:49:27,542 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731422967229.df409d6fa4df5d8fd91534d2d90eda60. 2024-11-12T14:49:27,542 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731422967229.df409d6fa4df5d8fd91534d2d90eda60. after waiting 0 ms 2024-11-12T14:49:27,542 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731422967229.df409d6fa4df5d8fd91534d2d90eda60. 2024-11-12T14:49:27,542 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731422967229.df409d6fa4df5d8fd91534d2d90eda60. 2024-11-12T14:49:27,542 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for df409d6fa4df5d8fd91534d2d90eda60: Waiting for close lock at 1731422967542Disabling compacts and flushes for region at 1731422967542Disabling writes for close at 1731422967542Writing region close event to WAL at 1731422967542Closed at 1731422967542 2024-11-12T14:49:27,543 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => eaeef83a325d6b259b1b899f552ba8d0, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731422967229.eaeef83a325d6b259b1b899f552ba8d0.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 2024-11-12T14:49:27,546 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731422967229.f65e71cd750425cce9713e30a75c6002.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:27,547 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing f65e71cd750425cce9713e30a75c6002, disabling compactions & flushes 2024-11-12T14:49:27,547 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731422967229.f65e71cd750425cce9713e30a75c6002. 2024-11-12T14:49:27,547 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731422967229.f65e71cd750425cce9713e30a75c6002. 2024-11-12T14:49:27,547 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731422967229.f65e71cd750425cce9713e30a75c6002. after waiting 0 ms 2024-11-12T14:49:27,547 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731422967229.f65e71cd750425cce9713e30a75c6002. 2024-11-12T14:49:27,547 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731422967229.f65e71cd750425cce9713e30a75c6002. 2024-11-12T14:49:27,547 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for f65e71cd750425cce9713e30a75c6002: Waiting for close lock at 1731422967546Disabling compacts and flushes for region at 1731422967546Disabling writes for close at 1731422967547 (+1 ms)Writing region close event to WAL at 1731422967547Closed at 1731422967547 2024-11-12T14:49:27,548 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 51b19d6922dec6f0774f3d761593de8f, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731422967229.51b19d6922dec6f0774f3d761593de8f.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 2024-11-12T14:49:27,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T14:49:27,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741847_1023 (size=110) 2024-11-12T14:49:27,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741847_1023 (size=110) 2024-11-12T14:49:27,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741847_1023 (size=110) 2024-11-12T14:49:27,592 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:27,592 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 92f3e9baa717d57147319661f6f00e06, disabling compactions & flushes 2024-11-12T14:49:27,592 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06. 2024-11-12T14:49:27,592 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06. 2024-11-12T14:49:27,592 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06. after waiting 0 ms 2024-11-12T14:49:27,592 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06. 2024-11-12T14:49:27,592 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06. 2024-11-12T14:49:27,592 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 92f3e9baa717d57147319661f6f00e06: Waiting for close lock at 1731422967592Disabling compacts and flushes for region at 1731422967592Disabling writes for close at 1731422967592Writing region close event to WAL at 1731422967592Closed at 1731422967592 2024-11-12T14:49:27,593 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ac225c32dbf496a52e546c7e0b1a05af, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731422967229.ac225c32dbf496a52e546c7e0b1a05af.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 2024-11-12T14:49:27,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741849_1025 (size=110) 2024-11-12T14:49:27,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741849_1025 (size=110) 2024-11-12T14:49:27,609 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731422967229.51b19d6922dec6f0774f3d761593de8f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:27,609 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 51b19d6922dec6f0774f3d761593de8f, disabling compactions & flushes 2024-11-12T14:49:27,609 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731422967229.51b19d6922dec6f0774f3d761593de8f. 2024-11-12T14:49:27,609 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731422967229.51b19d6922dec6f0774f3d761593de8f. 2024-11-12T14:49:27,609 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731422967229.51b19d6922dec6f0774f3d761593de8f. after waiting 0 ms 2024-11-12T14:49:27,609 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731422967229.51b19d6922dec6f0774f3d761593de8f. 2024-11-12T14:49:27,609 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731422967229.51b19d6922dec6f0774f3d761593de8f. 2024-11-12T14:49:27,610 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 51b19d6922dec6f0774f3d761593de8f: Waiting for close lock at 1731422967609Disabling compacts and flushes for region at 1731422967609Disabling writes for close at 1731422967609Writing region close event to WAL at 1731422967609Closed at 1731422967609 2024-11-12T14:49:27,610 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => d7c3d3ae3b942f0414dcbb110be8bc0b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 2024-11-12T14:49:27,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741849_1025 (size=110) 2024-11-12T14:49:27,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741848_1024 (size=110) 2024-11-12T14:49:27,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741848_1024 (size=110) 2024-11-12T14:49:27,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741848_1024 (size=110) 2024-11-12T14:49:27,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741850_1026 (size=110) 2024-11-12T14:49:27,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741850_1026 (size=110) 2024-11-12T14:49:27,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741850_1026 (size=110) 2024-11-12T14:49:27,619 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731422967229.ac225c32dbf496a52e546c7e0b1a05af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:27,619 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing ac225c32dbf496a52e546c7e0b1a05af, disabling compactions & flushes 2024-11-12T14:49:27,619 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731422967229.ac225c32dbf496a52e546c7e0b1a05af. 2024-11-12T14:49:27,619 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731422967229.ac225c32dbf496a52e546c7e0b1a05af. 2024-11-12T14:49:27,619 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731422967229.ac225c32dbf496a52e546c7e0b1a05af. after waiting 0 ms 2024-11-12T14:49:27,619 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731422967229.ac225c32dbf496a52e546c7e0b1a05af. 2024-11-12T14:49:27,619 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731422967229.ac225c32dbf496a52e546c7e0b1a05af. 2024-11-12T14:49:27,619 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for ac225c32dbf496a52e546c7e0b1a05af: Waiting for close lock at 1731422967619Disabling compacts and flushes for region at 1731422967619Disabling writes for close at 1731422967619Writing region close event to WAL at 1731422967619Closed at 1731422967619 2024-11-12T14:49:27,620 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bb0382e4edc08a7c43f0571deb2374a4, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731422967229.bb0382e4edc08a7c43f0571deb2374a4.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 2024-11-12T14:49:27,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741851_1027 (size=110) 2024-11-12T14:49:27,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741851_1027 (size=110) 2024-11-12T14:49:27,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741851_1027 (size=110) 2024-11-12T14:49:27,632 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:27,632 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing d7c3d3ae3b942f0414dcbb110be8bc0b, disabling compactions & flushes 2024-11-12T14:49:27,632 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b. 2024-11-12T14:49:27,633 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b. 2024-11-12T14:49:27,633 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b. after waiting 0 ms 2024-11-12T14:49:27,633 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b. 2024-11-12T14:49:27,633 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b. 2024-11-12T14:49:27,633 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for d7c3d3ae3b942f0414dcbb110be8bc0b: Waiting for close lock at 1731422967632Disabling compacts and flushes for region at 1731422967632Disabling writes for close at 1731422967633 (+1 ms)Writing region close event to WAL at 1731422967633Closed at 1731422967633 2024-11-12T14:49:27,634 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => b419710b4e7b088b11da63dff9b7760e, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731422967229.b419710b4e7b088b11da63dff9b7760e.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 2024-11-12T14:49:27,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741853_1029 (size=110) 2024-11-12T14:49:27,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741853_1029 (size=110) 2024-11-12T14:49:27,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741852_1028 (size=110) 2024-11-12T14:49:27,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741853_1029 (size=110) 2024-11-12T14:49:27,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741852_1028 (size=110) 2024-11-12T14:49:27,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741852_1028 (size=110) 2024-11-12T14:49:27,661 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731422967229.bb0382e4edc08a7c43f0571deb2374a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:27,661 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731422967229.b419710b4e7b088b11da63dff9b7760e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:27,661 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing bb0382e4edc08a7c43f0571deb2374a4, disabling compactions & flushes 2024-11-12T14:49:27,661 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing b419710b4e7b088b11da63dff9b7760e, disabling compactions & flushes 2024-11-12T14:49:27,661 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731422967229.b419710b4e7b088b11da63dff9b7760e. 2024-11-12T14:49:27,661 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731422967229.bb0382e4edc08a7c43f0571deb2374a4. 2024-11-12T14:49:27,661 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731422967229.b419710b4e7b088b11da63dff9b7760e. 2024-11-12T14:49:27,661 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731422967229.bb0382e4edc08a7c43f0571deb2374a4. 2024-11-12T14:49:27,661 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731422967229.b419710b4e7b088b11da63dff9b7760e. after waiting 0 ms 2024-11-12T14:49:27,661 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731422967229.bb0382e4edc08a7c43f0571deb2374a4. after waiting 0 ms 2024-11-12T14:49:27,662 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731422967229.b419710b4e7b088b11da63dff9b7760e. 2024-11-12T14:49:27,662 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731422967229.b419710b4e7b088b11da63dff9b7760e. 2024-11-12T14:49:27,662 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for b419710b4e7b088b11da63dff9b7760e: Waiting for close lock at 1731422967661Disabling compacts and flushes for region at 1731422967661Disabling writes for close at 1731422967662 (+1 ms)Writing region close event to WAL at 1731422967662Closed at 1731422967662 2024-11-12T14:49:27,662 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731422967229.bb0382e4edc08a7c43f0571deb2374a4. 2024-11-12T14:49:27,662 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731422967229.bb0382e4edc08a7c43f0571deb2374a4. 2024-11-12T14:49:27,662 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for bb0382e4edc08a7c43f0571deb2374a4: Waiting for close lock at 1731422967661Disabling compacts and flushes for region at 1731422967661Disabling writes for close at 1731422967662 (+1 ms)Writing region close event to WAL at 1731422967662Closed at 1731422967662 2024-11-12T14:49:27,663 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 2cc2e73ccc39f945b27a1d50dd2694bd, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 2024-11-12T14:49:27,663 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a2cc208a6afe25c72a2045fb03e0ecb1, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731422967229.a2cc208a6afe25c72a2045fb03e0ecb1.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 2024-11-12T14:49:27,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741855_1031 (size=110) 2024-11-12T14:49:27,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741855_1031 (size=110) 2024-11-12T14:49:27,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741855_1031 (size=110) 2024-11-12T14:49:27,708 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:27,708 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 2cc2e73ccc39f945b27a1d50dd2694bd, disabling compactions & flushes 2024-11-12T14:49:27,708 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd. 2024-11-12T14:49:27,708 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd. 2024-11-12T14:49:27,708 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd. after waiting 0 ms 2024-11-12T14:49:27,708 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd. 2024-11-12T14:49:27,708 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd. 2024-11-12T14:49:27,709 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 2cc2e73ccc39f945b27a1d50dd2694bd: Waiting for close lock at 1731422967708Disabling compacts and flushes for region at 1731422967708Disabling writes for close at 1731422967708Writing region close event to WAL at 1731422967708Closed at 1731422967708 2024-11-12T14:49:27,710 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 8b5e68238ca126132dbf55f0fbf38047, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731422967229.8b5e68238ca126132dbf55f0fbf38047.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 2024-11-12T14:49:27,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741854_1030 (size=110) 2024-11-12T14:49:27,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741854_1030 (size=110) 2024-11-12T14:49:27,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741854_1030 (size=110) 2024-11-12T14:49:27,724 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731422967229.a2cc208a6afe25c72a2045fb03e0ecb1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:27,725 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing a2cc208a6afe25c72a2045fb03e0ecb1, disabling compactions & flushes 2024-11-12T14:49:27,725 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731422967229.a2cc208a6afe25c72a2045fb03e0ecb1. 2024-11-12T14:49:27,725 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731422967229.a2cc208a6afe25c72a2045fb03e0ecb1. 2024-11-12T14:49:27,725 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731422967229.a2cc208a6afe25c72a2045fb03e0ecb1. after waiting 0 ms 2024-11-12T14:49:27,725 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731422967229.a2cc208a6afe25c72a2045fb03e0ecb1. 2024-11-12T14:49:27,725 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731422967229.a2cc208a6afe25c72a2045fb03e0ecb1. 2024-11-12T14:49:27,725 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for a2cc208a6afe25c72a2045fb03e0ecb1: Waiting for close lock at 1731422967725Disabling compacts and flushes for region at 1731422967725Disabling writes for close at 1731422967725Writing region close event to WAL at 1731422967725Closed at 1731422967725 2024-11-12T14:49:27,726 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2d931ca9d93687bd79e96c94cb16690b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 2024-11-12T14:49:27,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741856_1032 (size=110) 2024-11-12T14:49:27,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741856_1032 (size=110) 2024-11-12T14:49:27,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741856_1032 (size=110) 2024-11-12T14:49:27,746 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731422967229.8b5e68238ca126132dbf55f0fbf38047.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:27,747 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 8b5e68238ca126132dbf55f0fbf38047, disabling compactions & flushes 2024-11-12T14:49:27,747 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731422967229.8b5e68238ca126132dbf55f0fbf38047. 2024-11-12T14:49:27,747 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731422967229.8b5e68238ca126132dbf55f0fbf38047. 2024-11-12T14:49:27,747 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731422967229.8b5e68238ca126132dbf55f0fbf38047. after waiting 0 ms 2024-11-12T14:49:27,747 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731422967229.8b5e68238ca126132dbf55f0fbf38047. 2024-11-12T14:49:27,747 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731422967229.8b5e68238ca126132dbf55f0fbf38047. 2024-11-12T14:49:27,747 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 8b5e68238ca126132dbf55f0fbf38047: Waiting for close lock at 1731422967747Disabling compacts and flushes for region at 1731422967747Disabling writes for close at 1731422967747Writing region close event to WAL at 1731422967747Closed at 1731422967747 2024-11-12T14:49:27,748 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => efc23c8cc5ebe3a5c96ef5d199223832, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731422967229.efc23c8cc5ebe3a5c96ef5d199223832.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 2024-11-12T14:49:27,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741857_1033 (size=110) 2024-11-12T14:49:27,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741857_1033 (size=110) 2024-11-12T14:49:27,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741857_1033 (size=110) 2024-11-12T14:49:27,814 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:27,815 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 2d931ca9d93687bd79e96c94cb16690b, disabling compactions & flushes 2024-11-12T14:49:27,815 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b. 2024-11-12T14:49:27,815 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b. 2024-11-12T14:49:27,815 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b. after waiting 0 ms 2024-11-12T14:49:27,815 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b. 2024-11-12T14:49:27,815 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b. 2024-11-12T14:49:27,815 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2d931ca9d93687bd79e96c94cb16690b: Waiting for close lock at 1731422967815Disabling compacts and flushes for region at 1731422967815Disabling writes for close at 1731422967815Writing region close event to WAL at 1731422967815Closed at 1731422967815 2024-11-12T14:49:27,816 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2b04a6d83bd2998d75a1743bdf61651e, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731422967229.2b04a6d83bd2998d75a1743bdf61651e.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 2024-11-12T14:49:27,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741858_1034 (size=110) 2024-11-12T14:49:27,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741858_1034 (size=110) 2024-11-12T14:49:27,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741858_1034 (size=110) 2024-11-12T14:49:27,827 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731422967229.efc23c8cc5ebe3a5c96ef5d199223832.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:27,827 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing efc23c8cc5ebe3a5c96ef5d199223832, disabling compactions & flushes 2024-11-12T14:49:27,827 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731422967229.efc23c8cc5ebe3a5c96ef5d199223832. 2024-11-12T14:49:27,827 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731422967229.efc23c8cc5ebe3a5c96ef5d199223832. 2024-11-12T14:49:27,827 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731422967229.efc23c8cc5ebe3a5c96ef5d199223832. after waiting 0 ms 2024-11-12T14:49:27,828 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731422967229.efc23c8cc5ebe3a5c96ef5d199223832. 2024-11-12T14:49:27,828 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731422967229.efc23c8cc5ebe3a5c96ef5d199223832. 2024-11-12T14:49:27,828 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for efc23c8cc5ebe3a5c96ef5d199223832: Waiting for close lock at 1731422967827Disabling compacts and flushes for region at 1731422967827Disabling writes for close at 1731422967828 (+1 ms)Writing region close event to WAL at 1731422967828Closed at 1731422967828 2024-11-12T14:49:27,828 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 9a6f7a226796c8ff7e21a40d463cd27b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 2024-11-12T14:49:27,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741859_1035 (size=110) 2024-11-12T14:49:27,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741859_1035 (size=110) 2024-11-12T14:49:27,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741859_1035 (size=110) 2024-11-12T14:49:27,863 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731422967229.2b04a6d83bd2998d75a1743bdf61651e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:27,863 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing 2b04a6d83bd2998d75a1743bdf61651e, disabling compactions & flushes 2024-11-12T14:49:27,863 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731422967229.2b04a6d83bd2998d75a1743bdf61651e. 2024-11-12T14:49:27,864 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731422967229.2b04a6d83bd2998d75a1743bdf61651e. 2024-11-12T14:49:27,864 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731422967229.2b04a6d83bd2998d75a1743bdf61651e. after waiting 0 ms 2024-11-12T14:49:27,864 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731422967229.2b04a6d83bd2998d75a1743bdf61651e. 2024-11-12T14:49:27,864 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731422967229.2b04a6d83bd2998d75a1743bdf61651e. 2024-11-12T14:49:27,864 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2b04a6d83bd2998d75a1743bdf61651e: Waiting for close lock at 1731422967863Disabling compacts and flushes for region at 1731422967863Disabling writes for close at 1731422967864 (+1 ms)Writing region close event to WAL at 1731422967864Closed at 1731422967864 2024-11-12T14:49:27,865 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => aa9c293d6596b8e2756cafba09b3d7dd, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731422967229.aa9c293d6596b8e2756cafba09b3d7dd.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 2024-11-12T14:49:27,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741860_1036 (size=110) 2024-11-12T14:49:27,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741860_1036 (size=110) 2024-11-12T14:49:27,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741860_1036 (size=110) 2024-11-12T14:49:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T14:49:27,889 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:27,890 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 9a6f7a226796c8ff7e21a40d463cd27b, disabling compactions & flushes 2024-11-12T14:49:27,890 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b. 2024-11-12T14:49:27,890 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b. 2024-11-12T14:49:27,890 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b. after waiting 0 ms 2024-11-12T14:49:27,890 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b. 2024-11-12T14:49:27,890 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b. 2024-11-12T14:49:27,890 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 9a6f7a226796c8ff7e21a40d463cd27b: Waiting for close lock at 1731422967890Disabling compacts and flushes for region at 1731422967890Disabling writes for close at 1731422967890Writing region close event to WAL at 1731422967890Closed at 1731422967890 2024-11-12T14:49:27,891 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 95e5de757651e0aa099998abaa3b598f, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731422967229.95e5de757651e0aa099998abaa3b598f.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 2024-11-12T14:49:27,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741861_1037 (size=110) 2024-11-12T14:49:27,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741861_1037 (size=110) 2024-11-12T14:49:27,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741861_1037 (size=110) 2024-11-12T14:49:27,927 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731422967229.aa9c293d6596b8e2756cafba09b3d7dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:27,927 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing aa9c293d6596b8e2756cafba09b3d7dd, disabling compactions & flushes 2024-11-12T14:49:27,927 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731422967229.aa9c293d6596b8e2756cafba09b3d7dd. 2024-11-12T14:49:27,927 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731422967229.aa9c293d6596b8e2756cafba09b3d7dd. 2024-11-12T14:49:27,928 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731422967229.aa9c293d6596b8e2756cafba09b3d7dd. after waiting 0 ms 2024-11-12T14:49:27,928 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731422967229.aa9c293d6596b8e2756cafba09b3d7dd. 2024-11-12T14:49:27,928 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731422967229.aa9c293d6596b8e2756cafba09b3d7dd. 2024-11-12T14:49:27,928 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for aa9c293d6596b8e2756cafba09b3d7dd: Waiting for close lock at 1731422967927Disabling compacts and flushes for region at 1731422967927Disabling writes for close at 1731422967928 (+1 ms)Writing region close event to WAL at 1731422967928Closed at 1731422967928 2024-11-12T14:49:27,929 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e41036448a3677ed52f58f536690bf38, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050 2024-11-12T14:49:27,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741862_1038 (size=110) 2024-11-12T14:49:27,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741862_1038 (size=110) 2024-11-12T14:49:27,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741862_1038 (size=110) 2024-11-12T14:49:27,944 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731422967229.95e5de757651e0aa099998abaa3b598f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:27,945 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1722): Closing 95e5de757651e0aa099998abaa3b598f, disabling compactions & flushes 2024-11-12T14:49:27,945 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731422967229.95e5de757651e0aa099998abaa3b598f. 2024-11-12T14:49:27,945 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731422967229.95e5de757651e0aa099998abaa3b598f. 2024-11-12T14:49:27,945 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731422967229.95e5de757651e0aa099998abaa3b598f. after waiting 0 ms 2024-11-12T14:49:27,945 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731422967229.95e5de757651e0aa099998abaa3b598f. 2024-11-12T14:49:27,945 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731422967229.95e5de757651e0aa099998abaa3b598f. 2024-11-12T14:49:27,945 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-2 {}] regionserver.HRegion(1676): Region close journal for 95e5de757651e0aa099998abaa3b598f: Waiting for close lock at 1731422967945Disabling compacts and flushes for region at 1731422967945Disabling writes for close at 1731422967945Writing region close event to WAL at 1731422967945Closed at 1731422967945 2024-11-12T14:49:27,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741863_1039 (size=107) 2024-11-12T14:49:27,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741863_1039 (size=107) 2024-11-12T14:49:27,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741863_1039 (size=107) 2024-11-12T14:49:27,965 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:27,965 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1722): Closing e41036448a3677ed52f58f536690bf38, disabling compactions & flushes 2024-11-12T14:49:27,965 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38. 2024-11-12T14:49:27,966 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38. 2024-11-12T14:49:27,966 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38. after waiting 0 ms 2024-11-12T14:49:27,966 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38. 2024-11-12T14:49:27,966 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38. 2024-11-12T14:49:27,966 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-0 {}] regionserver.HRegion(1676): Region close journal for e41036448a3677ed52f58f536690bf38: Waiting for close lock at 1731422967965Disabling compacts and flushes for region at 1731422967965Disabling writes for close at 1731422967966 (+1 ms)Writing region close event to WAL at 1731422967966Closed at 1731422967966 2024-11-12T14:49:28,013 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731422967229.eaeef83a325d6b259b1b899f552ba8d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:28,013 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1722): Closing eaeef83a325d6b259b1b899f552ba8d0, disabling compactions & flushes 2024-11-12T14:49:28,013 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731422967229.eaeef83a325d6b259b1b899f552ba8d0. 2024-11-12T14:49:28,014 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731422967229.eaeef83a325d6b259b1b899f552ba8d0. 2024-11-12T14:49:28,014 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731422967229.eaeef83a325d6b259b1b899f552ba8d0. after waiting 0 ms 2024-11-12T14:49:28,014 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731422967229.eaeef83a325d6b259b1b899f552ba8d0. 2024-11-12T14:49:28,014 INFO [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731422967229.eaeef83a325d6b259b1b899f552ba8d0. 2024-11-12T14:49:28,014 DEBUG [RegionOpenAndInit-testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true-pool-1 {}] regionserver.HRegion(1676): Region close journal for eaeef83a325d6b259b1b899f552ba8d0: Waiting for close lock at 1731422968013Disabling compacts and flushes for region at 1731422968013Disabling writes for close at 1731422968014 (+1 ms)Writing region close event to WAL at 1731422968014Closed at 1731422968014 2024-11-12T14:49:28,019 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T14:49:28,027 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731422968019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731422968019"}]},"ts":"1731422968019"} 2024-11-12T14:49:28,027 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419.","families":{"info":[{"qualifier":"regioninfo","vlen":106,"tag":[],"timestamp":"1731422968019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731422968019"}]},"ts":"1731422968019"} 2024-11-12T14:49:28,027 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731422967229.489408be5f35a96bee1ee0f68f064599.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731422968019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731422968019"}]},"ts":"1731422968019"} 2024-11-12T14:49:28,027 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731422968019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731422968019"}]},"ts":"1731422968019"} 2024-11-12T14:49:28,028 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731422967229.cf0006d9b8777107f458b7008fc42ba9.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731422968019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731422968019"}]},"ts":"1731422968019"} 2024-11-12T14:49:28,028 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731422967229.59fa7126e2ade1f9daaaeb8bd38b09e4.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731422968019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731422968019"}]},"ts":"1731422968019"} 2024-11-12T14:49:28,028 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731422968019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731422968019"}]},"ts":"1731422968019"} 2024-11-12T14:49:28,028 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731422967229.df409d6fa4df5d8fd91534d2d90eda60.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731422968019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731422968019"}]},"ts":"1731422968019"} 2024-11-12T14:49:28,029 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731422967229.f65e71cd750425cce9713e30a75c6002.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731422968019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731422968019"}]},"ts":"1731422968019"} 2024-11-12T14:49:28,029 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731422968019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731422968019"}]},"ts":"1731422968019"} 2024-11-12T14:49:28,029 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731422967229.51b19d6922dec6f0774f3d761593de8f.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731422968019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731422968019"}]},"ts":"1731422968019"} 2024-11-12T14:49:28,029 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731422967229.ac225c32dbf496a52e546c7e0b1a05af.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731422968019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731422968019"}]},"ts":"1731422968019"} 2024-11-12T14:49:28,030 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731422968019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731422968019"}]},"ts":"1731422968019"} 2024-11-12T14:49:28,030 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731422967229.b419710b4e7b088b11da63dff9b7760e.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731422968019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731422968019"}]},"ts":"1731422968019"} 2024-11-12T14:49:28,030 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731422967229.bb0382e4edc08a7c43f0571deb2374a4.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731422968019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731422968019"}]},"ts":"1731422968019"} 2024-11-12T14:49:28,030 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731422968019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731422968019"}]},"ts":"1731422968019"} 2024-11-12T14:49:28,030 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731422967229.a2cc208a6afe25c72a2045fb03e0ecb1.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731422968019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731422968019"}]},"ts":"1731422968019"} 2024-11-12T14:49:28,030 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731422967229.8b5e68238ca126132dbf55f0fbf38047.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731422968019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731422968019"}]},"ts":"1731422968019"} 2024-11-12T14:49:28,031 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731422968019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731422968019"}]},"ts":"1731422968019"} 2024-11-12T14:49:28,031 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731422967229.efc23c8cc5ebe3a5c96ef5d199223832.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731422968019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731422968019"}]},"ts":"1731422968019"} 2024-11-12T14:49:28,031 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731422967229.2b04a6d83bd2998d75a1743bdf61651e.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731422968019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731422968019"}]},"ts":"1731422968019"} 2024-11-12T14:49:28,031 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731422968019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731422968019"}]},"ts":"1731422968019"} 2024-11-12T14:49:28,031 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731422967229.aa9c293d6596b8e2756cafba09b3d7dd.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731422968019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731422968019"}]},"ts":"1731422968019"} 2024-11-12T14:49:28,032 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731422967229.95e5de757651e0aa099998abaa3b598f.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731422968019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731422968019"}]},"ts":"1731422968019"} 2024-11-12T14:49:28,032 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38.","families":{"info":[{"qualifier":"regioninfo","vlen":106,"tag":[],"timestamp":"1731422968019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731422968019"}]},"ts":"1731422968019"} 2024-11-12T14:49:28,032 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731422967229.eaeef83a325d6b259b1b899f552ba8d0.","families":{"info":[{"qualifier":"regioninfo","vlen":109,"tag":[],"timestamp":"1731422968019"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731422968019"}]},"ts":"1731422968019"} 2024-11-12T14:49:28,093 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-12T14:49:28,096 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T14:49:28,099 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731422968096"}]},"ts":"1731422968096"} 2024-11-12T14:49:28,104 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, state=ENABLING in hbase:meta 2024-11-12T14:49:28,106 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {3945e1ca6f47=0} racks are {/default-rack=0} 2024-11-12T14:49:28,155 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T14:49:28,155 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T14:49:28,155 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-12T14:49:28,155 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T14:49:28,155 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T14:49:28,155 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-12T14:49:28,155 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T14:49:28,155 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T14:49:28,155 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-12T14:49:28,155 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T14:49:28,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f943ce043844f379c1b7ecbf76b5a419, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=489408be5f35a96bee1ee0f68f064599, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=06aa3b2e56ae4404b602d6fa98eabe33, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=cf0006d9b8777107f458b7008fc42ba9, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d3c298441d993e84c2d9e579914483db, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=59fa7126e2ade1f9daaaeb8bd38b09e4, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4936dfde0b7af7ea99d36c9e0c24fefc, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f65e71cd750425cce9713e30a75c6002, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=df409d6fa4df5d8fd91534d2d90eda60, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=92f3e9baa717d57147319661f6f00e06, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=eaeef83a325d6b259b1b899f552ba8d0, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=51b19d6922dec6f0774f3d761593de8f, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ac225c32dbf496a52e546c7e0b1a05af, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d7c3d3ae3b942f0414dcbb110be8bc0b, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bb0382e4edc08a7c43f0571deb2374a4, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b419710b4e7b088b11da63dff9b7760e, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2cc2e73ccc39f945b27a1d50dd2694bd, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a2cc208a6afe25c72a2045fb03e0ecb1, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8b5e68238ca126132dbf55f0fbf38047, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2d931ca9d93687bd79e96c94cb16690b, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=efc23c8cc5ebe3a5c96ef5d199223832, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2b04a6d83bd2998d75a1743bdf61651e, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9a6f7a226796c8ff7e21a40d463cd27b, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aa9c293d6596b8e2756cafba09b3d7dd, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=95e5de757651e0aa099998abaa3b598f, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e41036448a3677ed52f58f536690bf38, ASSIGN}] 2024-11-12T14:49:28,176 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e41036448a3677ed52f58f536690bf38, ASSIGN 2024-11-12T14:49:28,179 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e41036448a3677ed52f58f536690bf38, ASSIGN; state=OFFLINE, location=3945e1ca6f47,37185,1731422964568; forceNewPlan=false, retain=false 2024-11-12T14:49:28,181 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=95e5de757651e0aa099998abaa3b598f, ASSIGN 2024-11-12T14:49:28,183 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=95e5de757651e0aa099998abaa3b598f, ASSIGN; state=OFFLINE, location=3945e1ca6f47,35473,1731422964757; forceNewPlan=false, retain=false 2024-11-12T14:49:28,187 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aa9c293d6596b8e2756cafba09b3d7dd, ASSIGN 2024-11-12T14:49:28,189 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aa9c293d6596b8e2756cafba09b3d7dd, ASSIGN; state=OFFLINE, location=3945e1ca6f47,45845,1731422964694; forceNewPlan=false, retain=false 2024-11-12T14:49:28,193 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9a6f7a226796c8ff7e21a40d463cd27b, ASSIGN 2024-11-12T14:49:28,195 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9a6f7a226796c8ff7e21a40d463cd27b, ASSIGN; state=OFFLINE, location=3945e1ca6f47,37185,1731422964568; forceNewPlan=false, retain=false 2024-11-12T14:49:28,197 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2b04a6d83bd2998d75a1743bdf61651e, ASSIGN 2024-11-12T14:49:28,199 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2b04a6d83bd2998d75a1743bdf61651e, ASSIGN; state=OFFLINE, location=3945e1ca6f47,35473,1731422964757; forceNewPlan=false, retain=false 2024-11-12T14:49:28,201 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=efc23c8cc5ebe3a5c96ef5d199223832, ASSIGN 2024-11-12T14:49:28,202 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=efc23c8cc5ebe3a5c96ef5d199223832, ASSIGN; state=OFFLINE, location=3945e1ca6f47,45845,1731422964694; forceNewPlan=false, retain=false 2024-11-12T14:49:28,205 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2d931ca9d93687bd79e96c94cb16690b, ASSIGN 2024-11-12T14:49:28,208 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2d931ca9d93687bd79e96c94cb16690b, ASSIGN; state=OFFLINE, location=3945e1ca6f47,37185,1731422964568; forceNewPlan=false, retain=false 2024-11-12T14:49:28,210 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8b5e68238ca126132dbf55f0fbf38047, ASSIGN 2024-11-12T14:49:28,213 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8b5e68238ca126132dbf55f0fbf38047, ASSIGN; state=OFFLINE, location=3945e1ca6f47,35473,1731422964757; forceNewPlan=false, retain=false 2024-11-12T14:49:28,216 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a2cc208a6afe25c72a2045fb03e0ecb1, ASSIGN 2024-11-12T14:49:28,219 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a2cc208a6afe25c72a2045fb03e0ecb1, ASSIGN; state=OFFLINE, location=3945e1ca6f47,45845,1731422964694; forceNewPlan=false, retain=false 2024-11-12T14:49:28,222 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2cc2e73ccc39f945b27a1d50dd2694bd, ASSIGN 2024-11-12T14:49:28,224 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2cc2e73ccc39f945b27a1d50dd2694bd, ASSIGN; state=OFFLINE, location=3945e1ca6f47,37185,1731422964568; forceNewPlan=false, retain=false 2024-11-12T14:49:28,226 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b419710b4e7b088b11da63dff9b7760e, ASSIGN 2024-11-12T14:49:28,228 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b419710b4e7b088b11da63dff9b7760e, ASSIGN; state=OFFLINE, location=3945e1ca6f47,45845,1731422964694; forceNewPlan=false, retain=false 2024-11-12T14:49:28,229 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bb0382e4edc08a7c43f0571deb2374a4, ASSIGN 2024-11-12T14:49:28,231 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bb0382e4edc08a7c43f0571deb2374a4, ASSIGN; state=OFFLINE, location=3945e1ca6f47,35473,1731422964757; forceNewPlan=false, retain=false 2024-11-12T14:49:28,233 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d7c3d3ae3b942f0414dcbb110be8bc0b, ASSIGN 2024-11-12T14:49:28,236 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d7c3d3ae3b942f0414dcbb110be8bc0b, ASSIGN; state=OFFLINE, location=3945e1ca6f47,37185,1731422964568; forceNewPlan=false, retain=false 2024-11-12T14:49:28,238 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ac225c32dbf496a52e546c7e0b1a05af, ASSIGN 2024-11-12T14:49:28,240 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ac225c32dbf496a52e546c7e0b1a05af, ASSIGN; state=OFFLINE, location=3945e1ca6f47,35473,1731422964757; forceNewPlan=false, retain=false 2024-11-12T14:49:28,243 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=51b19d6922dec6f0774f3d761593de8f, ASSIGN 2024-11-12T14:49:28,246 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=51b19d6922dec6f0774f3d761593de8f, ASSIGN; state=OFFLINE, location=3945e1ca6f47,45845,1731422964694; forceNewPlan=false, retain=false 2024-11-12T14:49:28,248 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=eaeef83a325d6b259b1b899f552ba8d0, ASSIGN 2024-11-12T14:49:28,250 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=eaeef83a325d6b259b1b899f552ba8d0, ASSIGN; state=OFFLINE, location=3945e1ca6f47,45845,1731422964694; forceNewPlan=false, retain=false 2024-11-12T14:49:28,252 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=92f3e9baa717d57147319661f6f00e06, ASSIGN 2024-11-12T14:49:28,255 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=92f3e9baa717d57147319661f6f00e06, ASSIGN; state=OFFLINE, location=3945e1ca6f47,37185,1731422964568; forceNewPlan=false, retain=false 2024-11-12T14:49:28,258 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=df409d6fa4df5d8fd91534d2d90eda60, ASSIGN 2024-11-12T14:49:28,261 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=df409d6fa4df5d8fd91534d2d90eda60, ASSIGN; state=OFFLINE, location=3945e1ca6f47,45845,1731422964694; forceNewPlan=false, retain=false 2024-11-12T14:49:28,264 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f65e71cd750425cce9713e30a75c6002, ASSIGN 2024-11-12T14:49:28,267 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f65e71cd750425cce9713e30a75c6002, ASSIGN; state=OFFLINE, location=3945e1ca6f47,35473,1731422964757; forceNewPlan=false, retain=false 2024-11-12T14:49:28,269 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4936dfde0b7af7ea99d36c9e0c24fefc, ASSIGN 2024-11-12T14:49:28,271 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4936dfde0b7af7ea99d36c9e0c24fefc, ASSIGN; state=OFFLINE, location=3945e1ca6f47,37185,1731422964568; forceNewPlan=false, retain=false 2024-11-12T14:49:28,274 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=59fa7126e2ade1f9daaaeb8bd38b09e4, ASSIGN 2024-11-12T14:49:28,276 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=59fa7126e2ade1f9daaaeb8bd38b09e4, ASSIGN; state=OFFLINE, location=3945e1ca6f47,35473,1731422964757; forceNewPlan=false, retain=false 2024-11-12T14:49:28,280 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d3c298441d993e84c2d9e579914483db, ASSIGN 2024-11-12T14:49:28,283 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d3c298441d993e84c2d9e579914483db, ASSIGN; state=OFFLINE, location=3945e1ca6f47,37185,1731422964568; forceNewPlan=false, retain=false 2024-11-12T14:49:28,286 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=cf0006d9b8777107f458b7008fc42ba9, ASSIGN 2024-11-12T14:49:28,288 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=cf0006d9b8777107f458b7008fc42ba9, ASSIGN; state=OFFLINE, location=3945e1ca6f47,45845,1731422964694; forceNewPlan=false, retain=false 2024-11-12T14:49:28,290 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=06aa3b2e56ae4404b602d6fa98eabe33, ASSIGN 2024-11-12T14:49:28,292 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=06aa3b2e56ae4404b602d6fa98eabe33, ASSIGN; state=OFFLINE, location=3945e1ca6f47,37185,1731422964568; forceNewPlan=false, retain=false 2024-11-12T14:49:28,296 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=489408be5f35a96bee1ee0f68f064599, ASSIGN 2024-11-12T14:49:28,299 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=489408be5f35a96bee1ee0f68f064599, ASSIGN; state=OFFLINE, location=3945e1ca6f47,35473,1731422964757; forceNewPlan=false, retain=false 2024-11-12T14:49:28,305 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f943ce043844f379c1b7ecbf76b5a419, ASSIGN 2024-11-12T14:49:28,307 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f943ce043844f379c1b7ecbf76b5a419, ASSIGN; state=OFFLINE, location=3945e1ca6f47,45845,1731422964694; forceNewPlan=false, retain=false 2024-11-12T14:49:28,332 INFO [3945e1ca6f47:38303 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-12T14:49:28,333 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=aa9c293d6596b8e2756cafba09b3d7dd, regionState=OPENING, regionLocation=3945e1ca6f47,45845,1731422964694 2024-11-12T14:49:28,334 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=efc23c8cc5ebe3a5c96ef5d199223832, regionState=OPENING, regionLocation=3945e1ca6f47,45845,1731422964694 2024-11-12T14:49:28,335 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=a2cc208a6afe25c72a2045fb03e0ecb1, regionState=OPENING, regionLocation=3945e1ca6f47,45845,1731422964694 2024-11-12T14:49:28,336 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=b419710b4e7b088b11da63dff9b7760e, regionState=OPENING, regionLocation=3945e1ca6f47,45845,1731422964694 2024-11-12T14:49:28,337 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=51b19d6922dec6f0774f3d761593de8f, regionState=OPENING, regionLocation=3945e1ca6f47,45845,1731422964694 2024-11-12T14:49:28,338 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aa9c293d6596b8e2756cafba09b3d7dd, ASSIGN because future has completed 2024-11-12T14:49:28,338 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=eaeef83a325d6b259b1b899f552ba8d0, regionState=OPENING, regionLocation=3945e1ca6f47,45845,1731422964694 2024-11-12T14:49:28,339 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=df409d6fa4df5d8fd91534d2d90eda60, regionState=OPENING, regionLocation=3945e1ca6f47,45845,1731422964694 2024-11-12T14:49:28,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa9c293d6596b8e2756cafba09b3d7dd, server=3945e1ca6f47,45845,1731422964694}] 2024-11-12T14:49:28,341 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=efc23c8cc5ebe3a5c96ef5d199223832, ASSIGN because future has completed 2024-11-12T14:49:28,346 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a2cc208a6afe25c72a2045fb03e0ecb1, ASSIGN because future has completed 2024-11-12T14:49:28,352 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b419710b4e7b088b11da63dff9b7760e, ASSIGN because future has completed 2024-11-12T14:49:28,354 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=51b19d6922dec6f0774f3d761593de8f, ASSIGN because future has completed 2024-11-12T14:49:28,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 51b19d6922dec6f0774f3d761593de8f, server=3945e1ca6f47,45845,1731422964694}] 2024-11-12T14:49:28,360 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=eaeef83a325d6b259b1b899f552ba8d0, ASSIGN because future has completed 2024-11-12T14:49:28,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=df409d6fa4df5d8fd91534d2d90eda60, ASSIGN because future has completed 2024-11-12T14:49:28,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure df409d6fa4df5d8fd91534d2d90eda60, server=3945e1ca6f47,45845,1731422964694}] 2024-11-12T14:49:28,374 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure eaeef83a325d6b259b1b899f552ba8d0, server=3945e1ca6f47,45845,1731422964694}] 2024-11-12T14:49:28,381 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure b419710b4e7b088b11da63dff9b7760e, server=3945e1ca6f47,45845,1731422964694}] 2024-11-12T14:49:28,388 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2cc208a6afe25c72a2045fb03e0ecb1, server=3945e1ca6f47,45845,1731422964694}] 2024-11-12T14:49:28,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T14:49:28,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure efc23c8cc5ebe3a5c96ef5d199223832, server=3945e1ca6f47,45845,1731422964694}] 2024-11-12T14:49:28,421 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=cf0006d9b8777107f458b7008fc42ba9, regionState=OPENING, regionLocation=3945e1ca6f47,45845,1731422964694 2024-11-12T14:49:28,422 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f943ce043844f379c1b7ecbf76b5a419, regionState=OPENING, regionLocation=3945e1ca6f47,45845,1731422964694 2024-11-12T14:49:28,422 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=e41036448a3677ed52f58f536690bf38, regionState=OPENING, regionLocation=3945e1ca6f47,37185,1731422964568 2024-11-12T14:49:28,424 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=9a6f7a226796c8ff7e21a40d463cd27b, regionState=OPENING, regionLocation=3945e1ca6f47,37185,1731422964568 2024-11-12T14:49:28,424 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=2d931ca9d93687bd79e96c94cb16690b, regionState=OPENING, regionLocation=3945e1ca6f47,37185,1731422964568 2024-11-12T14:49:28,425 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=2cc2e73ccc39f945b27a1d50dd2694bd, regionState=OPENING, regionLocation=3945e1ca6f47,37185,1731422964568 2024-11-12T14:49:28,426 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=d7c3d3ae3b942f0414dcbb110be8bc0b, regionState=OPENING, regionLocation=3945e1ca6f47,37185,1731422964568 2024-11-12T14:49:28,427 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=92f3e9baa717d57147319661f6f00e06, regionState=OPENING, regionLocation=3945e1ca6f47,37185,1731422964568 2024-11-12T14:49:28,428 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=4936dfde0b7af7ea99d36c9e0c24fefc, regionState=OPENING, regionLocation=3945e1ca6f47,37185,1731422964568 2024-11-12T14:49:28,428 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=d3c298441d993e84c2d9e579914483db, regionState=OPENING, regionLocation=3945e1ca6f47,37185,1731422964568 2024-11-12T14:49:28,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=cf0006d9b8777107f458b7008fc42ba9, ASSIGN because future has completed 2024-11-12T14:49:28,430 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=06aa3b2e56ae4404b602d6fa98eabe33, regionState=OPENING, regionLocation=3945e1ca6f47,37185,1731422964568 2024-11-12T14:49:28,432 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=95e5de757651e0aa099998abaa3b598f, regionState=OPENING, regionLocation=3945e1ca6f47,35473,1731422964757 2024-11-12T14:49:28,433 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=2b04a6d83bd2998d75a1743bdf61651e, regionState=OPENING, regionLocation=3945e1ca6f47,35473,1731422964757 2024-11-12T14:49:28,434 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=8b5e68238ca126132dbf55f0fbf38047, regionState=OPENING, regionLocation=3945e1ca6f47,35473,1731422964757 2024-11-12T14:49:28,435 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=bb0382e4edc08a7c43f0571deb2374a4, regionState=OPENING, regionLocation=3945e1ca6f47,35473,1731422964757 2024-11-12T14:49:28,436 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=ac225c32dbf496a52e546c7e0b1a05af, regionState=OPENING, regionLocation=3945e1ca6f47,35473,1731422964757 2024-11-12T14:49:28,437 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=f65e71cd750425cce9713e30a75c6002, regionState=OPENING, regionLocation=3945e1ca6f47,35473,1731422964757 2024-11-12T14:49:28,441 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf0006d9b8777107f458b7008fc42ba9, server=3945e1ca6f47,45845,1731422964694}] 2024-11-12T14:49:28,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f943ce043844f379c1b7ecbf76b5a419, ASSIGN because future has completed 2024-11-12T14:49:28,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e41036448a3677ed52f58f536690bf38, ASSIGN because future has completed 2024-11-12T14:49:28,449 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9a6f7a226796c8ff7e21a40d463cd27b, ASSIGN because future has completed 2024-11-12T14:49:28,451 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2d931ca9d93687bd79e96c94cb16690b, ASSIGN because future has completed 2024-11-12T14:49:28,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d931ca9d93687bd79e96c94cb16690b, server=3945e1ca6f47,37185,1731422964568}] 2024-11-12T14:49:28,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2cc2e73ccc39f945b27a1d50dd2694bd, ASSIGN because future has completed 2024-11-12T14:49:28,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d7c3d3ae3b942f0414dcbb110be8bc0b, ASSIGN because future has completed 2024-11-12T14:49:28,460 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=92f3e9baa717d57147319661f6f00e06, ASSIGN because future has completed 2024-11-12T14:49:28,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92f3e9baa717d57147319661f6f00e06, server=3945e1ca6f47,37185,1731422964568}] 2024-11-12T14:49:28,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4936dfde0b7af7ea99d36c9e0c24fefc, ASSIGN because future has completed 2024-11-12T14:49:28,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4936dfde0b7af7ea99d36c9e0c24fefc, server=3945e1ca6f47,37185,1731422964568}] 2024-11-12T14:49:28,477 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d3c298441d993e84c2d9e579914483db, ASSIGN because future has completed 2024-11-12T14:49:28,480 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=06aa3b2e56ae4404b602d6fa98eabe33, ASSIGN because future has completed 2024-11-12T14:49:28,482 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=95e5de757651e0aa099998abaa3b598f, ASSIGN because future has completed 2024-11-12T14:49:28,484 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2b04a6d83bd2998d75a1743bdf61651e, ASSIGN because future has completed 2024-11-12T14:49:28,486 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8b5e68238ca126132dbf55f0fbf38047, ASSIGN because future has completed 2024-11-12T14:49:28,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b5e68238ca126132dbf55f0fbf38047, server=3945e1ca6f47,35473,1731422964757}] 2024-11-12T14:49:28,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bb0382e4edc08a7c43f0571deb2374a4, ASSIGN because future has completed 2024-11-12T14:49:28,491 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ac225c32dbf496a52e546c7e0b1a05af, ASSIGN because future has completed 2024-11-12T14:49:28,493 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f65e71cd750425cce9713e30a75c6002, ASSIGN because future has completed 2024-11-12T14:49:28,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure f65e71cd750425cce9713e30a75c6002, server=3945e1ca6f47,35473,1731422964757}] 2024-11-12T14:49:28,510 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T14:49:28,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure ac225c32dbf496a52e546c7e0b1a05af, server=3945e1ca6f47,35473,1731422964757}] 2024-11-12T14:49:28,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure bb0382e4edc08a7c43f0571deb2374a4, server=3945e1ca6f47,35473,1731422964757}] 2024-11-12T14:49:28,522 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60807, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T14:49:28,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b04a6d83bd2998d75a1743bdf61651e, server=3945e1ca6f47,35473,1731422964757}] 2024-11-12T14:49:28,534 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 95e5de757651e0aa099998abaa3b598f, server=3945e1ca6f47,35473,1731422964757}] 2024-11-12T14:49:28,537 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731422967229.aa9c293d6596b8e2756cafba09b3d7dd. 2024-11-12T14:49:28,537 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => aa9c293d6596b8e2756cafba09b3d7dd, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731422967229.aa9c293d6596b8e2756cafba09b3d7dd.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-12T14:49:28,538 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true aa9c293d6596b8e2756cafba09b3d7dd 2024-11-12T14:49:28,539 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731422967229.aa9c293d6596b8e2756cafba09b3d7dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:28,539 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for aa9c293d6596b8e2756cafba09b3d7dd 2024-11-12T14:49:28,539 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for aa9c293d6596b8e2756cafba09b3d7dd 2024-11-12T14:49:28,542 INFO [StoreOpener-aa9c293d6596b8e2756cafba09b3d7dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region aa9c293d6596b8e2756cafba09b3d7dd 2024-11-12T14:49:28,542 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06aa3b2e56ae4404b602d6fa98eabe33, server=3945e1ca6f47,37185,1731422964568}] 2024-11-12T14:49:28,545 INFO [StoreOpener-aa9c293d6596b8e2756cafba09b3d7dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa9c293d6596b8e2756cafba09b3d7dd columnFamilyName colfamily11 2024-11-12T14:49:28,545 DEBUG [StoreOpener-aa9c293d6596b8e2756cafba09b3d7dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,547 INFO [StoreOpener-aa9c293d6596b8e2756cafba09b3d7dd-1 {}] regionserver.HStore(327): Store=aa9c293d6596b8e2756cafba09b3d7dd/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,547 INFO [StoreOpener-aa9c293d6596b8e2756cafba09b3d7dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region aa9c293d6596b8e2756cafba09b3d7dd 2024-11-12T14:49:28,549 INFO [StoreOpener-aa9c293d6596b8e2756cafba09b3d7dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa9c293d6596b8e2756cafba09b3d7dd columnFamilyName colfamily21 2024-11-12T14:49:28,549 DEBUG [StoreOpener-aa9c293d6596b8e2756cafba09b3d7dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,550 INFO [StoreOpener-aa9c293d6596b8e2756cafba09b3d7dd-1 {}] regionserver.HStore(327): Store=aa9c293d6596b8e2756cafba09b3d7dd/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,551 INFO [StoreOpener-aa9c293d6596b8e2756cafba09b3d7dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region aa9c293d6596b8e2756cafba09b3d7dd 2024-11-12T14:49:28,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3c298441d993e84c2d9e579914483db, server=3945e1ca6f47,37185,1731422964568}] 2024-11-12T14:49:28,553 INFO [StoreOpener-aa9c293d6596b8e2756cafba09b3d7dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa9c293d6596b8e2756cafba09b3d7dd columnFamilyName colfamily31 2024-11-12T14:49:28,553 DEBUG [StoreOpener-aa9c293d6596b8e2756cafba09b3d7dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,554 INFO [StoreOpener-aa9c293d6596b8e2756cafba09b3d7dd-1 {}] regionserver.HStore(327): Store=aa9c293d6596b8e2756cafba09b3d7dd/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,555 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for aa9c293d6596b8e2756cafba09b3d7dd 2024-11-12T14:49:28,557 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa9c293d6596b8e2756cafba09b3d7dd 2024-11-12T14:49:28,558 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa9c293d6596b8e2756cafba09b3d7dd 2024-11-12T14:49:28,560 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for aa9c293d6596b8e2756cafba09b3d7dd 2024-11-12T14:49:28,560 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for aa9c293d6596b8e2756cafba09b3d7dd 2024-11-12T14:49:28,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure d7c3d3ae3b942f0414dcbb110be8bc0b, server=3945e1ca6f47,37185,1731422964568}] 2024-11-12T14:49:28,561 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:49:28,564 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for aa9c293d6596b8e2756cafba09b3d7dd 2024-11-12T14:49:28,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2cc2e73ccc39f945b27a1d50dd2694bd, server=3945e1ca6f47,37185,1731422964568}] 2024-11-12T14:49:28,570 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa9c293d6596b8e2756cafba09b3d7dd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:49:28,571 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened aa9c293d6596b8e2756cafba09b3d7dd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64306168, jitterRate=-0.04176342487335205}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:49:28,571 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aa9c293d6596b8e2756cafba09b3d7dd 2024-11-12T14:49:28,573 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for aa9c293d6596b8e2756cafba09b3d7dd: Running coprocessor pre-open hook at 1731422968539Writing region info on filesystem at 1731422968539Initializing all the Stores at 1731422968541 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968541Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968542 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968542Cleaning up temporary data from old regions at 1731422968560 (+18 ms)Running coprocessor post-open hooks at 1731422968571 (+11 ms)Region opened successfully at 1731422968573 (+2 ms) 2024-11-12T14:49:28,574 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9a6f7a226796c8ff7e21a40d463cd27b, server=3945e1ca6f47,37185,1731422964568}] 2024-11-12T14:49:28,581 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731422967229.aa9c293d6596b8e2756cafba09b3d7dd., pid=31, masterSystemTime=1731422968510 2024-11-12T14:49:28,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure e41036448a3677ed52f58f536690bf38, server=3945e1ca6f47,37185,1731422964568}] 2024-11-12T14:49:28,596 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731422967229.aa9c293d6596b8e2756cafba09b3d7dd. 2024-11-12T14:49:28,597 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731422967229.aa9c293d6596b8e2756cafba09b3d7dd. 2024-11-12T14:49:28,597 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731422967229.eaeef83a325d6b259b1b899f552ba8d0. 2024-11-12T14:49:28,597 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => eaeef83a325d6b259b1b899f552ba8d0, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731422967229.eaeef83a325d6b259b1b899f552ba8d0.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-12T14:49:28,598 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true eaeef83a325d6b259b1b899f552ba8d0 2024-11-12T14:49:28,598 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731422967229.eaeef83a325d6b259b1b899f552ba8d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:28,598 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for eaeef83a325d6b259b1b899f552ba8d0 2024-11-12T14:49:28,598 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for eaeef83a325d6b259b1b899f552ba8d0 2024-11-12T14:49:28,605 INFO [StoreOpener-eaeef83a325d6b259b1b899f552ba8d0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region eaeef83a325d6b259b1b899f552ba8d0 2024-11-12T14:49:28,605 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=aa9c293d6596b8e2756cafba09b3d7dd, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,45845,1731422964694 2024-11-12T14:49:28,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f943ce043844f379c1b7ecbf76b5a419, server=3945e1ca6f47,45845,1731422964694}] 2024-11-12T14:49:28,609 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa9c293d6596b8e2756cafba09b3d7dd, server=3945e1ca6f47,45845,1731422964694 because future has completed 2024-11-12T14:49:28,609 INFO [StoreOpener-eaeef83a325d6b259b1b899f552ba8d0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eaeef83a325d6b259b1b899f552ba8d0 columnFamilyName colfamily11 2024-11-12T14:49:28,609 DEBUG [StoreOpener-eaeef83a325d6b259b1b899f552ba8d0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,610 INFO [StoreOpener-eaeef83a325d6b259b1b899f552ba8d0-1 {}] regionserver.HStore(327): Store=eaeef83a325d6b259b1b899f552ba8d0/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,610 INFO [StoreOpener-eaeef83a325d6b259b1b899f552ba8d0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region eaeef83a325d6b259b1b899f552ba8d0 2024-11-12T14:49:28,613 INFO [StoreOpener-eaeef83a325d6b259b1b899f552ba8d0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eaeef83a325d6b259b1b899f552ba8d0 columnFamilyName colfamily21 2024-11-12T14:49:28,613 DEBUG [StoreOpener-eaeef83a325d6b259b1b899f552ba8d0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,614 INFO [StoreOpener-eaeef83a325d6b259b1b899f552ba8d0-1 {}] regionserver.HStore(327): Store=eaeef83a325d6b259b1b899f552ba8d0/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,614 INFO [StoreOpener-eaeef83a325d6b259b1b899f552ba8d0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region eaeef83a325d6b259b1b899f552ba8d0 2024-11-12T14:49:28,616 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38303 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3945e1ca6f47,45845,1731422964694, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aa9c293d6596b8e2756cafba09b3d7dd. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T14:49:28,617 INFO [StoreOpener-eaeef83a325d6b259b1b899f552ba8d0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eaeef83a325d6b259b1b899f552ba8d0 columnFamilyName colfamily31 2024-11-12T14:49:28,617 DEBUG [StoreOpener-eaeef83a325d6b259b1b899f552ba8d0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,618 INFO [StoreOpener-eaeef83a325d6b259b1b899f552ba8d0-1 {}] regionserver.HStore(327): Store=eaeef83a325d6b259b1b899f552ba8d0/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,619 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for eaeef83a325d6b259b1b899f552ba8d0 2024-11-12T14:49:28,620 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/eaeef83a325d6b259b1b899f552ba8d0 2024-11-12T14:49:28,621 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/eaeef83a325d6b259b1b899f552ba8d0 2024-11-12T14:49:28,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=28 2024-11-12T14:49:28,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure aa9c293d6596b8e2756cafba09b3d7dd, server=3945e1ca6f47,45845,1731422964694 in 277 msec 2024-11-12T14:49:28,623 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc. 2024-11-12T14:49:28,623 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 4936dfde0b7af7ea99d36c9e0c24fefc, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-12T14:49:28,624 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 4936dfde0b7af7ea99d36c9e0c24fefc 2024-11-12T14:49:28,624 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:28,624 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 4936dfde0b7af7ea99d36c9e0c24fefc 2024-11-12T14:49:28,624 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 4936dfde0b7af7ea99d36c9e0c24fefc 2024-11-12T14:49:28,625 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for eaeef83a325d6b259b1b899f552ba8d0 2024-11-12T14:49:28,625 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for eaeef83a325d6b259b1b899f552ba8d0 2024-11-12T14:49:28,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=aa9c293d6596b8e2756cafba09b3d7dd, ASSIGN in 464 msec 2024-11-12T14:49:28,626 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:49:28,626 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=59fa7126e2ade1f9daaaeb8bd38b09e4, regionState=OPENING, regionLocation=3945e1ca6f47,35473,1731422964757 2024-11-12T14:49:28,626 INFO [StoreOpener-4936dfde0b7af7ea99d36c9e0c24fefc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 4936dfde0b7af7ea99d36c9e0c24fefc 2024-11-12T14:49:28,627 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=489408be5f35a96bee1ee0f68f064599, regionState=OPENING, regionLocation=3945e1ca6f47,35473,1731422964757 2024-11-12T14:49:28,630 INFO [StoreOpener-4936dfde0b7af7ea99d36c9e0c24fefc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4936dfde0b7af7ea99d36c9e0c24fefc columnFamilyName colfamily11 2024-11-12T14:49:28,630 DEBUG [StoreOpener-4936dfde0b7af7ea99d36c9e0c24fefc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,631 INFO [StoreOpener-4936dfde0b7af7ea99d36c9e0c24fefc-1 {}] regionserver.HStore(327): Store=4936dfde0b7af7ea99d36c9e0c24fefc/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,631 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=59fa7126e2ade1f9daaaeb8bd38b09e4, ASSIGN because future has completed 2024-11-12T14:49:28,631 INFO [StoreOpener-4936dfde0b7af7ea99d36c9e0c24fefc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 4936dfde0b7af7ea99d36c9e0c24fefc 2024-11-12T14:49:28,631 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for eaeef83a325d6b259b1b899f552ba8d0 2024-11-12T14:49:28,631 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59fa7126e2ade1f9daaaeb8bd38b09e4, server=3945e1ca6f47,35473,1731422964757}] 2024-11-12T14:49:28,633 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=489408be5f35a96bee1ee0f68f064599, ASSIGN because future has completed 2024-11-12T14:49:28,634 INFO [StoreOpener-4936dfde0b7af7ea99d36c9e0c24fefc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4936dfde0b7af7ea99d36c9e0c24fefc columnFamilyName colfamily21 2024-11-12T14:49:28,634 DEBUG [StoreOpener-4936dfde0b7af7ea99d36c9e0c24fefc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,635 INFO [StoreOpener-4936dfde0b7af7ea99d36c9e0c24fefc-1 {}] regionserver.HStore(327): Store=4936dfde0b7af7ea99d36c9e0c24fefc/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,635 INFO [StoreOpener-4936dfde0b7af7ea99d36c9e0c24fefc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 4936dfde0b7af7ea99d36c9e0c24fefc 2024-11-12T14:49:28,638 INFO [StoreOpener-4936dfde0b7af7ea99d36c9e0c24fefc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4936dfde0b7af7ea99d36c9e0c24fefc columnFamilyName colfamily31 2024-11-12T14:49:28,638 DEBUG [StoreOpener-4936dfde0b7af7ea99d36c9e0c24fefc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 489408be5f35a96bee1ee0f68f064599, server=3945e1ca6f47,35473,1731422964757}] 2024-11-12T14:49:28,639 INFO [StoreOpener-4936dfde0b7af7ea99d36c9e0c24fefc-1 {}] regionserver.HStore(327): Store=4936dfde0b7af7ea99d36c9e0c24fefc/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,640 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 4936dfde0b7af7ea99d36c9e0c24fefc 2024-11-12T14:49:28,641 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4936dfde0b7af7ea99d36c9e0c24fefc 2024-11-12T14:49:28,642 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T14:49:28,642 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4936dfde0b7af7ea99d36c9e0c24fefc 2024-11-12T14:49:28,644 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39249, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T14:49:28,646 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 4936dfde0b7af7ea99d36c9e0c24fefc 2024-11-12T14:49:28,646 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 4936dfde0b7af7ea99d36c9e0c24fefc 2024-11-12T14:49:28,648 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:49:28,650 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 4936dfde0b7af7ea99d36c9e0c24fefc 2024-11-12T14:49:28,653 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731422967229.2b04a6d83bd2998d75a1743bdf61651e. 2024-11-12T14:49:28,653 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/eaeef83a325d6b259b1b899f552ba8d0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:49:28,654 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 2b04a6d83bd2998d75a1743bdf61651e, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731422967229.2b04a6d83bd2998d75a1743bdf61651e.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-12T14:49:28,654 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2b04a6d83bd2998d75a1743bdf61651e 2024-11-12T14:49:28,654 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731422967229.2b04a6d83bd2998d75a1743bdf61651e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:28,654 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 2b04a6d83bd2998d75a1743bdf61651e 2024-11-12T14:49:28,655 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 2b04a6d83bd2998d75a1743bdf61651e 2024-11-12T14:49:28,655 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened eaeef83a325d6b259b1b899f552ba8d0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59901382, jitterRate=-0.1073998510837555}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:49:28,655 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eaeef83a325d6b259b1b899f552ba8d0 2024-11-12T14:49:28,655 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for eaeef83a325d6b259b1b899f552ba8d0: Running coprocessor pre-open hook at 1731422968599Writing region info on filesystem at 1731422968599Initializing all the Stores at 1731422968601 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968601Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968605 (+4 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968605Cleaning up temporary data from old regions at 1731422968625 (+20 ms)Running coprocessor post-open hooks at 1731422968655 (+30 ms)Region opened successfully at 1731422968655 2024-11-12T14:49:28,656 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731422967229.eaeef83a325d6b259b1b899f552ba8d0., pid=34, masterSystemTime=1731422968510 2024-11-12T14:49:28,659 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4936dfde0b7af7ea99d36c9e0c24fefc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:49:28,660 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731422967229.eaeef83a325d6b259b1b899f552ba8d0. 2024-11-12T14:49:28,660 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 4936dfde0b7af7ea99d36c9e0c24fefc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63686049, jitterRate=-0.051003918051719666}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:49:28,661 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731422967229.eaeef83a325d6b259b1b899f552ba8d0. 2024-11-12T14:49:28,661 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4936dfde0b7af7ea99d36c9e0c24fefc 2024-11-12T14:49:28,661 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731422967229.51b19d6922dec6f0774f3d761593de8f. 2024-11-12T14:49:28,661 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 4936dfde0b7af7ea99d36c9e0c24fefc: Running coprocessor pre-open hook at 1731422968624Writing region info on filesystem at 1731422968625 (+1 ms)Initializing all the Stores at 1731422968626 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968626Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968626Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968626Cleaning up temporary data from old regions at 1731422968646 (+20 ms)Running coprocessor post-open hooks at 1731422968661 (+15 ms)Region opened successfully at 1731422968661 2024-11-12T14:49:28,661 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 51b19d6922dec6f0774f3d761593de8f, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731422967229.51b19d6922dec6f0774f3d761593de8f.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-12T14:49:28,662 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 51b19d6922dec6f0774f3d761593de8f 2024-11-12T14:49:28,662 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731422967229.51b19d6922dec6f0774f3d761593de8f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:28,662 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 51b19d6922dec6f0774f3d761593de8f 2024-11-12T14:49:28,662 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 51b19d6922dec6f0774f3d761593de8f 2024-11-12T14:49:28,662 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc., pid=41, masterSystemTime=1731422968613 2024-11-12T14:49:28,664 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=eaeef83a325d6b259b1b899f552ba8d0, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,45845,1731422964694 2024-11-12T14:49:28,666 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc. 2024-11-12T14:49:28,667 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc. 2024-11-12T14:49:28,667 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd. 2024-11-12T14:49:28,667 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 2cc2e73ccc39f945b27a1d50dd2694bd, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-12T14:49:28,668 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2cc2e73ccc39f945b27a1d50dd2694bd 2024-11-12T14:49:28,668 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:28,668 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure eaeef83a325d6b259b1b899f552ba8d0, server=3945e1ca6f47,45845,1731422964694 because future has completed 2024-11-12T14:49:28,668 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 2cc2e73ccc39f945b27a1d50dd2694bd 2024-11-12T14:49:28,668 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 2cc2e73ccc39f945b27a1d50dd2694bd 2024-11-12T14:49:28,669 INFO [StoreOpener-2b04a6d83bd2998d75a1743bdf61651e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 2b04a6d83bd2998d75a1743bdf61651e 2024-11-12T14:49:28,670 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=4936dfde0b7af7ea99d36c9e0c24fefc, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,37185,1731422964568 2024-11-12T14:49:28,672 INFO [StoreOpener-2b04a6d83bd2998d75a1743bdf61651e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b04a6d83bd2998d75a1743bdf61651e columnFamilyName colfamily11 2024-11-12T14:49:28,672 DEBUG [StoreOpener-2b04a6d83bd2998d75a1743bdf61651e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,673 INFO [StoreOpener-2b04a6d83bd2998d75a1743bdf61651e-1 {}] regionserver.HStore(327): Store=2b04a6d83bd2998d75a1743bdf61651e/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,674 INFO [StoreOpener-2b04a6d83bd2998d75a1743bdf61651e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 2b04a6d83bd2998d75a1743bdf61651e 2024-11-12T14:49:28,674 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4936dfde0b7af7ea99d36c9e0c24fefc, server=3945e1ca6f47,37185,1731422964568 because future has completed 2024-11-12T14:49:28,677 INFO [StoreOpener-2b04a6d83bd2998d75a1743bdf61651e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b04a6d83bd2998d75a1743bdf61651e columnFamilyName colfamily21 2024-11-12T14:49:28,677 DEBUG [StoreOpener-2b04a6d83bd2998d75a1743bdf61651e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,677 INFO [StoreOpener-51b19d6922dec6f0774f3d761593de8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 51b19d6922dec6f0774f3d761593de8f 2024-11-12T14:49:28,678 INFO [StoreOpener-2b04a6d83bd2998d75a1743bdf61651e-1 {}] regionserver.HStore(327): Store=2b04a6d83bd2998d75a1743bdf61651e/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,679 INFO [StoreOpener-2b04a6d83bd2998d75a1743bdf61651e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 2b04a6d83bd2998d75a1743bdf61651e 2024-11-12T14:49:28,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=15 2024-11-12T14:49:28,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure eaeef83a325d6b259b1b899f552ba8d0, server=3945e1ca6f47,45845,1731422964694 in 302 msec 2024-11-12T14:49:28,680 INFO [StoreOpener-51b19d6922dec6f0774f3d761593de8f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51b19d6922dec6f0774f3d761593de8f columnFamilyName colfamily11 2024-11-12T14:49:28,680 DEBUG [StoreOpener-51b19d6922dec6f0774f3d761593de8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,681 INFO [StoreOpener-51b19d6922dec6f0774f3d761593de8f-1 {}] regionserver.HStore(327): Store=51b19d6922dec6f0774f3d761593de8f/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,681 INFO [StoreOpener-2b04a6d83bd2998d75a1743bdf61651e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b04a6d83bd2998d75a1743bdf61651e columnFamilyName colfamily31 2024-11-12T14:49:28,681 DEBUG [StoreOpener-2b04a6d83bd2998d75a1743bdf61651e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,682 INFO [StoreOpener-51b19d6922dec6f0774f3d761593de8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 51b19d6922dec6f0774f3d761593de8f 2024-11-12T14:49:28,682 INFO [StoreOpener-2b04a6d83bd2998d75a1743bdf61651e-1 {}] regionserver.HStore(327): Store=2b04a6d83bd2998d75a1743bdf61651e/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,683 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 2b04a6d83bd2998d75a1743bdf61651e 2024-11-12T14:49:28,683 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=eaeef83a325d6b259b1b899f552ba8d0, ASSIGN in 522 msec 2024-11-12T14:49:28,684 INFO [StoreOpener-51b19d6922dec6f0774f3d761593de8f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51b19d6922dec6f0774f3d761593de8f columnFamilyName colfamily21 2024-11-12T14:49:28,684 DEBUG [StoreOpener-51b19d6922dec6f0774f3d761593de8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,685 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2b04a6d83bd2998d75a1743bdf61651e 2024-11-12T14:49:28,686 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2b04a6d83bd2998d75a1743bdf61651e 2024-11-12T14:49:28,686 INFO [StoreOpener-51b19d6922dec6f0774f3d761593de8f-1 {}] regionserver.HStore(327): Store=51b19d6922dec6f0774f3d761593de8f/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,687 INFO [StoreOpener-51b19d6922dec6f0774f3d761593de8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 51b19d6922dec6f0774f3d761593de8f 2024-11-12T14:49:28,689 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 2b04a6d83bd2998d75a1743bdf61651e 2024-11-12T14:49:28,689 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 2b04a6d83bd2998d75a1743bdf61651e 2024-11-12T14:49:28,690 INFO [StoreOpener-51b19d6922dec6f0774f3d761593de8f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51b19d6922dec6f0774f3d761593de8f columnFamilyName colfamily31 2024-11-12T14:49:28,690 DEBUG [StoreOpener-51b19d6922dec6f0774f3d761593de8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,691 INFO [StoreOpener-51b19d6922dec6f0774f3d761593de8f-1 {}] regionserver.HStore(327): Store=51b19d6922dec6f0774f3d761593de8f/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,691 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:49:28,691 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 51b19d6922dec6f0774f3d761593de8f 2024-11-12T14:49:28,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=11 2024-11-12T14:49:28,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 4936dfde0b7af7ea99d36c9e0c24fefc, server=3945e1ca6f47,37185,1731422964568 in 209 msec 2024-11-12T14:49:28,693 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/51b19d6922dec6f0774f3d761593de8f 2024-11-12T14:49:28,694 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/51b19d6922dec6f0774f3d761593de8f 2024-11-12T14:49:28,694 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 2b04a6d83bd2998d75a1743bdf61651e 2024-11-12T14:49:28,696 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 51b19d6922dec6f0774f3d761593de8f 2024-11-12T14:49:28,696 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 51b19d6922dec6f0774f3d761593de8f 2024-11-12T14:49:28,696 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4936dfde0b7af7ea99d36c9e0c24fefc, ASSIGN in 534 msec 2024-11-12T14:49:28,697 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:49:28,697 INFO [StoreOpener-2cc2e73ccc39f945b27a1d50dd2694bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 2cc2e73ccc39f945b27a1d50dd2694bd 2024-11-12T14:49:28,699 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2b04a6d83bd2998d75a1743bdf61651e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:49:28,699 INFO [StoreOpener-2cc2e73ccc39f945b27a1d50dd2694bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2cc2e73ccc39f945b27a1d50dd2694bd columnFamilyName colfamily11 2024-11-12T14:49:28,699 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 51b19d6922dec6f0774f3d761593de8f 2024-11-12T14:49:28,699 DEBUG [StoreOpener-2cc2e73ccc39f945b27a1d50dd2694bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,700 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 2b04a6d83bd2998d75a1743bdf61651e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71694774, jitterRate=0.0683353841304779}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:49:28,700 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2b04a6d83bd2998d75a1743bdf61651e 2024-11-12T14:49:28,700 INFO [StoreOpener-2cc2e73ccc39f945b27a1d50dd2694bd-1 {}] regionserver.HStore(327): Store=2cc2e73ccc39f945b27a1d50dd2694bd/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,700 INFO [StoreOpener-2cc2e73ccc39f945b27a1d50dd2694bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 2cc2e73ccc39f945b27a1d50dd2694bd 2024-11-12T14:49:28,700 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 2b04a6d83bd2998d75a1743bdf61651e: Running coprocessor pre-open hook at 1731422968655Writing region info on filesystem at 1731422968655Initializing all the Stores at 1731422968659 (+4 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968659Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968669 (+10 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968669Cleaning up temporary data from old regions at 1731422968689 (+20 ms)Running coprocessor post-open hooks at 1731422968700 (+11 ms)Region opened successfully at 1731422968700 2024-11-12T14:49:28,702 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731422967229.2b04a6d83bd2998d75a1743bdf61651e., pid=46, masterSystemTime=1731422968641 2024-11-12T14:49:28,703 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/51b19d6922dec6f0774f3d761593de8f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:49:28,703 INFO [StoreOpener-2cc2e73ccc39f945b27a1d50dd2694bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2cc2e73ccc39f945b27a1d50dd2694bd columnFamilyName colfamily21 2024-11-12T14:49:28,703 DEBUG [StoreOpener-2cc2e73ccc39f945b27a1d50dd2694bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,703 INFO [StoreOpener-2cc2e73ccc39f945b27a1d50dd2694bd-1 {}] regionserver.HStore(327): Store=2cc2e73ccc39f945b27a1d50dd2694bd/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,703 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 51b19d6922dec6f0774f3d761593de8f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59161312, jitterRate=-0.11842775344848633}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:49:28,704 INFO [StoreOpener-2cc2e73ccc39f945b27a1d50dd2694bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 2cc2e73ccc39f945b27a1d50dd2694bd 2024-11-12T14:49:28,704 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 51b19d6922dec6f0774f3d761593de8f 2024-11-12T14:49:28,704 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 51b19d6922dec6f0774f3d761593de8f: Running coprocessor pre-open hook at 1731422968662Writing region info on filesystem at 1731422968662Initializing all the Stores at 1731422968665 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968665Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968677 (+12 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968677Cleaning up temporary data from old regions at 1731422968696 (+19 ms)Running coprocessor post-open hooks at 1731422968704 (+8 ms)Region opened successfully at 1731422968704 2024-11-12T14:49:28,706 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731422967229.51b19d6922dec6f0774f3d761593de8f., pid=32, masterSystemTime=1731422968510 2024-11-12T14:49:28,707 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731422967229.2b04a6d83bd2998d75a1743bdf61651e. 2024-11-12T14:49:28,707 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731422967229.2b04a6d83bd2998d75a1743bdf61651e. 2024-11-12T14:49:28,707 INFO [StoreOpener-2cc2e73ccc39f945b27a1d50dd2694bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2cc2e73ccc39f945b27a1d50dd2694bd columnFamilyName colfamily31 2024-11-12T14:49:28,707 DEBUG [StoreOpener-2cc2e73ccc39f945b27a1d50dd2694bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,707 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731422967229.bb0382e4edc08a7c43f0571deb2374a4. 2024-11-12T14:49:28,707 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => bb0382e4edc08a7c43f0571deb2374a4, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731422967229.bb0382e4edc08a7c43f0571deb2374a4.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-12T14:49:28,708 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true bb0382e4edc08a7c43f0571deb2374a4 2024-11-12T14:49:28,708 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731422967229.bb0382e4edc08a7c43f0571deb2374a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:28,708 INFO [StoreOpener-2cc2e73ccc39f945b27a1d50dd2694bd-1 {}] regionserver.HStore(327): Store=2cc2e73ccc39f945b27a1d50dd2694bd/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,708 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for bb0382e4edc08a7c43f0571deb2374a4 2024-11-12T14:49:28,708 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 2cc2e73ccc39f945b27a1d50dd2694bd 2024-11-12T14:49:28,709 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for bb0382e4edc08a7c43f0571deb2374a4 2024-11-12T14:49:28,709 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=2b04a6d83bd2998d75a1743bdf61651e, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,35473,1731422964757 2024-11-12T14:49:28,709 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731422967229.51b19d6922dec6f0774f3d761593de8f. 2024-11-12T14:49:28,709 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731422967229.51b19d6922dec6f0774f3d761593de8f. 2024-11-12T14:49:28,710 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731422967229.efc23c8cc5ebe3a5c96ef5d199223832. 2024-11-12T14:49:28,710 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => efc23c8cc5ebe3a5c96ef5d199223832, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731422967229.efc23c8cc5ebe3a5c96ef5d199223832.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-12T14:49:28,710 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true efc23c8cc5ebe3a5c96ef5d199223832 2024-11-12T14:49:28,711 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731422967229.efc23c8cc5ebe3a5c96ef5d199223832.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:28,711 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for efc23c8cc5ebe3a5c96ef5d199223832 2024-11-12T14:49:28,711 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for efc23c8cc5ebe3a5c96ef5d199223832 2024-11-12T14:49:28,712 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cc2e73ccc39f945b27a1d50dd2694bd 2024-11-12T14:49:28,714 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cc2e73ccc39f945b27a1d50dd2694bd 2024-11-12T14:49:28,715 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=51b19d6922dec6f0774f3d761593de8f, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,45845,1731422964694 2024-11-12T14:49:28,716 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 2cc2e73ccc39f945b27a1d50dd2694bd 2024-11-12T14:49:28,716 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b04a6d83bd2998d75a1743bdf61651e, server=3945e1ca6f47,35473,1731422964757 because future has completed 2024-11-12T14:49:28,716 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 2cc2e73ccc39f945b27a1d50dd2694bd 2024-11-12T14:49:28,717 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:49:28,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 51b19d6922dec6f0774f3d761593de8f, server=3945e1ca6f47,45845,1731422964694 because future has completed 2024-11-12T14:49:28,721 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 2cc2e73ccc39f945b27a1d50dd2694bd 2024-11-12T14:49:28,722 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38303 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3945e1ca6f47,45845,1731422964694, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=51b19d6922dec6f0774f3d761593de8f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T14:49:28,725 INFO [StoreOpener-bb0382e4edc08a7c43f0571deb2374a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region bb0382e4edc08a7c43f0571deb2374a4 2024-11-12T14:49:28,725 INFO [StoreOpener-efc23c8cc5ebe3a5c96ef5d199223832-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region efc23c8cc5ebe3a5c96ef5d199223832 2024-11-12T14:49:28,728 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cc2e73ccc39f945b27a1d50dd2694bd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:49:28,728 INFO [StoreOpener-bb0382e4edc08a7c43f0571deb2374a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bb0382e4edc08a7c43f0571deb2374a4 columnFamilyName colfamily11 2024-11-12T14:49:28,729 DEBUG [StoreOpener-bb0382e4edc08a7c43f0571deb2374a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,729 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 2cc2e73ccc39f945b27a1d50dd2694bd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70722643, jitterRate=0.05384950339794159}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:49:28,729 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2cc2e73ccc39f945b27a1d50dd2694bd 2024-11-12T14:49:28,729 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 2cc2e73ccc39f945b27a1d50dd2694bd: Running coprocessor pre-open hook at 1731422968668Writing region info on filesystem at 1731422968668Initializing all the Stores at 1731422968671 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968672 (+1 ms)Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968693 (+21 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968693Cleaning up temporary data from old regions at 1731422968716 (+23 ms)Running coprocessor post-open hooks at 1731422968729 (+13 ms)Region opened successfully at 1731422968729 2024-11-12T14:49:28,730 INFO [StoreOpener-bb0382e4edc08a7c43f0571deb2374a4-1 {}] regionserver.HStore(327): Store=bb0382e4edc08a7c43f0571deb2374a4/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,730 INFO [StoreOpener-bb0382e4edc08a7c43f0571deb2374a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region bb0382e4edc08a7c43f0571deb2374a4 2024-11-12T14:49:28,730 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=26 2024-11-12T14:49:28,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 2b04a6d83bd2998d75a1743bdf61651e, server=3945e1ca6f47,35473,1731422964757 in 195 msec 2024-11-12T14:49:28,732 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd., pid=51, masterSystemTime=1731422968613 2024-11-12T14:49:28,733 INFO [StoreOpener-bb0382e4edc08a7c43f0571deb2374a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bb0382e4edc08a7c43f0571deb2374a4 columnFamilyName colfamily21 2024-11-12T14:49:28,733 DEBUG [StoreOpener-bb0382e4edc08a7c43f0571deb2374a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2b04a6d83bd2998d75a1743bdf61651e, ASSIGN in 573 msec 2024-11-12T14:49:28,735 INFO [StoreOpener-bb0382e4edc08a7c43f0571deb2374a4-1 {}] regionserver.HStore(327): Store=bb0382e4edc08a7c43f0571deb2374a4/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,736 INFO [StoreOpener-bb0382e4edc08a7c43f0571deb2374a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region bb0382e4edc08a7c43f0571deb2374a4 2024-11-12T14:49:28,737 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd. 2024-11-12T14:49:28,737 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd. 2024-11-12T14:49:28,737 INFO [StoreOpener-efc23c8cc5ebe3a5c96ef5d199223832-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efc23c8cc5ebe3a5c96ef5d199223832 columnFamilyName colfamily11 2024-11-12T14:49:28,737 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b. 2024-11-12T14:49:28,737 DEBUG [StoreOpener-efc23c8cc5ebe3a5c96ef5d199223832-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,737 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => d7c3d3ae3b942f0414dcbb110be8bc0b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-12T14:49:28,738 INFO [StoreOpener-bb0382e4edc08a7c43f0571deb2374a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bb0382e4edc08a7c43f0571deb2374a4 columnFamilyName colfamily31 2024-11-12T14:49:28,738 DEBUG [StoreOpener-bb0382e4edc08a7c43f0571deb2374a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,738 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true d7c3d3ae3b942f0414dcbb110be8bc0b 2024-11-12T14:49:28,739 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:28,739 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for d7c3d3ae3b942f0414dcbb110be8bc0b 2024-11-12T14:49:28,739 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for d7c3d3ae3b942f0414dcbb110be8bc0b 2024-11-12T14:49:28,739 INFO [StoreOpener-efc23c8cc5ebe3a5c96ef5d199223832-1 {}] regionserver.HStore(327): Store=efc23c8cc5ebe3a5c96ef5d199223832/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,739 INFO [StoreOpener-bb0382e4edc08a7c43f0571deb2374a4-1 {}] regionserver.HStore(327): Store=bb0382e4edc08a7c43f0571deb2374a4/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,739 INFO [StoreOpener-efc23c8cc5ebe3a5c96ef5d199223832-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region efc23c8cc5ebe3a5c96ef5d199223832 2024-11-12T14:49:28,739 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for bb0382e4edc08a7c43f0571deb2374a4 2024-11-12T14:49:28,741 INFO [StoreOpener-d7c3d3ae3b942f0414dcbb110be8bc0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region d7c3d3ae3b942f0414dcbb110be8bc0b 2024-11-12T14:49:28,741 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bb0382e4edc08a7c43f0571deb2374a4 2024-11-12T14:49:28,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=16 2024-11-12T14:49:28,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 51b19d6922dec6f0774f3d761593de8f, server=3945e1ca6f47,45845,1731422964694 in 378 msec 2024-11-12T14:49:28,742 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bb0382e4edc08a7c43f0571deb2374a4 2024-11-12T14:49:28,743 INFO [StoreOpener-efc23c8cc5ebe3a5c96ef5d199223832-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efc23c8cc5ebe3a5c96ef5d199223832 columnFamilyName colfamily21 2024-11-12T14:49:28,743 DEBUG [StoreOpener-efc23c8cc5ebe3a5c96ef5d199223832-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,744 INFO [StoreOpener-d7c3d3ae3b942f0414dcbb110be8bc0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7c3d3ae3b942f0414dcbb110be8bc0b columnFamilyName colfamily11 2024-11-12T14:49:28,744 DEBUG [StoreOpener-d7c3d3ae3b942f0414dcbb110be8bc0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,744 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for bb0382e4edc08a7c43f0571deb2374a4 2024-11-12T14:49:28,744 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for bb0382e4edc08a7c43f0571deb2374a4 2024-11-12T14:49:28,745 INFO [StoreOpener-d7c3d3ae3b942f0414dcbb110be8bc0b-1 {}] regionserver.HStore(327): Store=d7c3d3ae3b942f0414dcbb110be8bc0b/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,745 INFO [StoreOpener-efc23c8cc5ebe3a5c96ef5d199223832-1 {}] regionserver.HStore(327): Store=efc23c8cc5ebe3a5c96ef5d199223832/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,745 INFO [StoreOpener-efc23c8cc5ebe3a5c96ef5d199223832-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region efc23c8cc5ebe3a5c96ef5d199223832 2024-11-12T14:49:28,745 INFO [StoreOpener-d7c3d3ae3b942f0414dcbb110be8bc0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region d7c3d3ae3b942f0414dcbb110be8bc0b 2024-11-12T14:49:28,746 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:49:28,747 INFO [StoreOpener-d7c3d3ae3b942f0414dcbb110be8bc0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7c3d3ae3b942f0414dcbb110be8bc0b columnFamilyName colfamily21 2024-11-12T14:49:28,747 DEBUG [StoreOpener-d7c3d3ae3b942f0414dcbb110be8bc0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,748 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=51b19d6922dec6f0774f3d761593de8f, ASSIGN in 584 msec 2024-11-12T14:49:28,748 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for bb0382e4edc08a7c43f0571deb2374a4 2024-11-12T14:49:28,749 INFO [StoreOpener-d7c3d3ae3b942f0414dcbb110be8bc0b-1 {}] regionserver.HStore(327): Store=d7c3d3ae3b942f0414dcbb110be8bc0b/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,749 INFO [StoreOpener-d7c3d3ae3b942f0414dcbb110be8bc0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region d7c3d3ae3b942f0414dcbb110be8bc0b 2024-11-12T14:49:28,751 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=2cc2e73ccc39f945b27a1d50dd2694bd, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,37185,1731422964568 2024-11-12T14:49:28,751 INFO [StoreOpener-d7c3d3ae3b942f0414dcbb110be8bc0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7c3d3ae3b942f0414dcbb110be8bc0b columnFamilyName colfamily31 2024-11-12T14:49:28,752 DEBUG [StoreOpener-d7c3d3ae3b942f0414dcbb110be8bc0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,753 INFO [StoreOpener-d7c3d3ae3b942f0414dcbb110be8bc0b-1 {}] regionserver.HStore(327): Store=d7c3d3ae3b942f0414dcbb110be8bc0b/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,753 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for d7c3d3ae3b942f0414dcbb110be8bc0b 2024-11-12T14:49:28,755 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d7c3d3ae3b942f0414dcbb110be8bc0b 2024-11-12T14:49:28,755 INFO [StoreOpener-efc23c8cc5ebe3a5c96ef5d199223832-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efc23c8cc5ebe3a5c96ef5d199223832 columnFamilyName colfamily31 2024-11-12T14:49:28,756 DEBUG [StoreOpener-efc23c8cc5ebe3a5c96ef5d199223832-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2cc2e73ccc39f945b27a1d50dd2694bd, server=3945e1ca6f47,37185,1731422964568 because future has completed 2024-11-12T14:49:28,756 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d7c3d3ae3b942f0414dcbb110be8bc0b 2024-11-12T14:49:28,757 INFO [StoreOpener-efc23c8cc5ebe3a5c96ef5d199223832-1 {}] regionserver.HStore(327): Store=efc23c8cc5ebe3a5c96ef5d199223832/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,758 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for d7c3d3ae3b942f0414dcbb110be8bc0b 2024-11-12T14:49:28,758 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for d7c3d3ae3b942f0414dcbb110be8bc0b 2024-11-12T14:49:28,758 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for efc23c8cc5ebe3a5c96ef5d199223832 2024-11-12T14:49:28,759 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:49:28,759 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/efc23c8cc5ebe3a5c96ef5d199223832 2024-11-12T14:49:28,760 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/efc23c8cc5ebe3a5c96ef5d199223832 2024-11-12T14:49:28,761 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for d7c3d3ae3b942f0414dcbb110be8bc0b 2024-11-12T14:49:28,763 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for efc23c8cc5ebe3a5c96ef5d199223832 2024-11-12T14:49:28,763 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for efc23c8cc5ebe3a5c96ef5d199223832 2024-11-12T14:49:28,764 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:49:28,766 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bb0382e4edc08a7c43f0571deb2374a4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:49:28,768 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for efc23c8cc5ebe3a5c96ef5d199223832 2024-11-12T14:49:28,768 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened bb0382e4edc08a7c43f0571deb2374a4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58933982, jitterRate=-0.12181523442268372}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:49:28,768 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bb0382e4edc08a7c43f0571deb2374a4 2024-11-12T14:49:28,768 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for bb0382e4edc08a7c43f0571deb2374a4: Running coprocessor pre-open hook at 1731422968710Writing region info on filesystem at 1731422968710Initializing all the Stores at 1731422968714 (+4 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968714Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968725 (+11 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968725Cleaning up temporary data from old regions at 1731422968744 (+19 ms)Running coprocessor post-open hooks at 1731422968768 (+24 ms)Region opened successfully at 1731422968768 2024-11-12T14:49:28,771 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731422967229.bb0382e4edc08a7c43f0571deb2374a4., pid=45, masterSystemTime=1731422968641 2024-11-12T14:49:28,771 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=21 2024-11-12T14:49:28,771 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 2cc2e73ccc39f945b27a1d50dd2694bd, server=3945e1ca6f47,37185,1731422964568 in 195 msec 2024-11-12T14:49:28,774 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731422967229.bb0382e4edc08a7c43f0571deb2374a4. 2024-11-12T14:49:28,774 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d7c3d3ae3b942f0414dcbb110be8bc0b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:49:28,774 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/efc23c8cc5ebe3a5c96ef5d199223832/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:49:28,774 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731422967229.bb0382e4edc08a7c43f0571deb2374a4. 2024-11-12T14:49:28,775 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731422967229.59fa7126e2ade1f9daaaeb8bd38b09e4. 2024-11-12T14:49:28,775 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 59fa7126e2ade1f9daaaeb8bd38b09e4, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731422967229.59fa7126e2ade1f9daaaeb8bd38b09e4.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-12T14:49:28,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2cc2e73ccc39f945b27a1d50dd2694bd, ASSIGN in 613 msec 2024-11-12T14:49:28,775 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened efc23c8cc5ebe3a5c96ef5d199223832; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64791610, jitterRate=-0.03452977538108826}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:49:28,775 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened d7c3d3ae3b942f0414dcbb110be8bc0b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72305210, jitterRate=0.0774315893650055}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:49:28,775 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for efc23c8cc5ebe3a5c96ef5d199223832 2024-11-12T14:49:28,775 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d7c3d3ae3b942f0414dcbb110be8bc0b 2024-11-12T14:49:28,776 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 59fa7126e2ade1f9daaaeb8bd38b09e4 2024-11-12T14:49:28,776 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for efc23c8cc5ebe3a5c96ef5d199223832: Running coprocessor pre-open hook at 1731422968711Writing region info on filesystem at 1731422968711Initializing all the Stores at 1731422968714 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968714Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968724 (+10 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968724Cleaning up temporary data from old regions at 1731422968763 (+39 ms)Running coprocessor post-open hooks at 1731422968775 (+12 ms)Region opened successfully at 1731422968776 (+1 ms) 2024-11-12T14:49:28,776 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for d7c3d3ae3b942f0414dcbb110be8bc0b: Running coprocessor pre-open hook at 1731422968739Writing region info on filesystem at 1731422968739Initializing all the Stores at 1731422968740 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968740Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968741 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968741Cleaning up temporary data from old regions at 1731422968758 (+17 ms)Running coprocessor post-open hooks at 1731422968775 (+17 ms)Region opened successfully at 1731422968776 (+1 ms) 2024-11-12T14:49:28,776 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731422967229.59fa7126e2ade1f9daaaeb8bd38b09e4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:28,776 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 59fa7126e2ade1f9daaaeb8bd38b09e4 2024-11-12T14:49:28,776 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 59fa7126e2ade1f9daaaeb8bd38b09e4 2024-11-12T14:49:28,777 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b., pid=50, masterSystemTime=1731422968613 2024-11-12T14:49:28,777 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731422967229.efc23c8cc5ebe3a5c96ef5d199223832., pid=37, masterSystemTime=1731422968510 2024-11-12T14:49:28,777 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=bb0382e4edc08a7c43f0571deb2374a4, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,35473,1731422964757 2024-11-12T14:49:28,779 INFO [StoreOpener-59fa7126e2ade1f9daaaeb8bd38b09e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 59fa7126e2ade1f9daaaeb8bd38b09e4 2024-11-12T14:49:28,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure bb0382e4edc08a7c43f0571deb2374a4, server=3945e1ca6f47,35473,1731422964757 because future has completed 2024-11-12T14:49:28,783 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731422967229.efc23c8cc5ebe3a5c96ef5d199223832. 2024-11-12T14:49:28,783 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731422967229.efc23c8cc5ebe3a5c96ef5d199223832. 2024-11-12T14:49:28,783 INFO [StoreOpener-59fa7126e2ade1f9daaaeb8bd38b09e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59fa7126e2ade1f9daaaeb8bd38b09e4 columnFamilyName colfamily11 2024-11-12T14:49:28,783 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731422967229.df409d6fa4df5d8fd91534d2d90eda60. 2024-11-12T14:49:28,783 DEBUG [StoreOpener-59fa7126e2ade1f9daaaeb8bd38b09e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,783 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => df409d6fa4df5d8fd91534d2d90eda60, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731422967229.df409d6fa4df5d8fd91534d2d90eda60.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-12T14:49:28,784 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true df409d6fa4df5d8fd91534d2d90eda60 2024-11-12T14:49:28,784 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731422967229.df409d6fa4df5d8fd91534d2d90eda60.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:28,784 INFO [StoreOpener-59fa7126e2ade1f9daaaeb8bd38b09e4-1 {}] regionserver.HStore(327): Store=59fa7126e2ade1f9daaaeb8bd38b09e4/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,784 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for df409d6fa4df5d8fd91534d2d90eda60 2024-11-12T14:49:28,784 INFO [StoreOpener-59fa7126e2ade1f9daaaeb8bd38b09e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 59fa7126e2ade1f9daaaeb8bd38b09e4 2024-11-12T14:49:28,784 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for df409d6fa4df5d8fd91534d2d90eda60 2024-11-12T14:49:28,786 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=efc23c8cc5ebe3a5c96ef5d199223832, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,45845,1731422964694 2024-11-12T14:49:28,787 INFO [StoreOpener-df409d6fa4df5d8fd91534d2d90eda60-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region df409d6fa4df5d8fd91534d2d90eda60 2024-11-12T14:49:28,787 INFO [StoreOpener-59fa7126e2ade1f9daaaeb8bd38b09e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59fa7126e2ade1f9daaaeb8bd38b09e4 columnFamilyName colfamily21 2024-11-12T14:49:28,788 DEBUG [StoreOpener-59fa7126e2ade1f9daaaeb8bd38b09e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,790 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b. 2024-11-12T14:49:28,790 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b. 2024-11-12T14:49:28,790 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db. 2024-11-12T14:49:28,790 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => d3c298441d993e84c2d9e579914483db, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-12T14:49:28,791 INFO [StoreOpener-df409d6fa4df5d8fd91534d2d90eda60-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df409d6fa4df5d8fd91534d2d90eda60 columnFamilyName colfamily11 2024-11-12T14:49:28,791 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true d3c298441d993e84c2d9e579914483db 2024-11-12T14:49:28,791 DEBUG [StoreOpener-df409d6fa4df5d8fd91534d2d90eda60-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,791 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:28,791 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for d3c298441d993e84c2d9e579914483db 2024-11-12T14:49:28,791 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for d3c298441d993e84c2d9e579914483db 2024-11-12T14:49:28,792 INFO [StoreOpener-59fa7126e2ade1f9daaaeb8bd38b09e4-1 {}] regionserver.HStore(327): Store=59fa7126e2ade1f9daaaeb8bd38b09e4/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,792 INFO [StoreOpener-59fa7126e2ade1f9daaaeb8bd38b09e4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 59fa7126e2ade1f9daaaeb8bd38b09e4 2024-11-12T14:49:28,792 INFO [StoreOpener-df409d6fa4df5d8fd91534d2d90eda60-1 {}] regionserver.HStore(327): Store=df409d6fa4df5d8fd91534d2d90eda60/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,793 INFO [StoreOpener-df409d6fa4df5d8fd91534d2d90eda60-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region df409d6fa4df5d8fd91534d2d90eda60 2024-11-12T14:49:28,795 INFO [StoreOpener-59fa7126e2ade1f9daaaeb8bd38b09e4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59fa7126e2ade1f9daaaeb8bd38b09e4 columnFamilyName colfamily31 2024-11-12T14:49:28,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure efc23c8cc5ebe3a5c96ef5d199223832, server=3945e1ca6f47,45845,1731422964694 because future has completed 2024-11-12T14:49:28,795 INFO [StoreOpener-df409d6fa4df5d8fd91534d2d90eda60-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df409d6fa4df5d8fd91534d2d90eda60 columnFamilyName colfamily21 2024-11-12T14:49:28,795 DEBUG [StoreOpener-59fa7126e2ade1f9daaaeb8bd38b09e4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,796 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=d7c3d3ae3b942f0414dcbb110be8bc0b, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,37185,1731422964568 2024-11-12T14:49:28,796 DEBUG [StoreOpener-df409d6fa4df5d8fd91534d2d90eda60-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,796 INFO [StoreOpener-59fa7126e2ade1f9daaaeb8bd38b09e4-1 {}] regionserver.HStore(327): Store=59fa7126e2ade1f9daaaeb8bd38b09e4/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,796 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 59fa7126e2ade1f9daaaeb8bd38b09e4 2024-11-12T14:49:28,798 INFO [StoreOpener-df409d6fa4df5d8fd91534d2d90eda60-1 {}] regionserver.HStore(327): Store=df409d6fa4df5d8fd91534d2d90eda60/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,798 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/59fa7126e2ade1f9daaaeb8bd38b09e4 2024-11-12T14:49:28,799 INFO [StoreOpener-df409d6fa4df5d8fd91534d2d90eda60-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region df409d6fa4df5d8fd91534d2d90eda60 2024-11-12T14:49:28,799 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/59fa7126e2ade1f9daaaeb8bd38b09e4 2024-11-12T14:49:28,801 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure d7c3d3ae3b942f0414dcbb110be8bc0b, server=3945e1ca6f47,37185,1731422964568 because future has completed 2024-11-12T14:49:28,802 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 59fa7126e2ade1f9daaaeb8bd38b09e4 2024-11-12T14:49:28,802 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 59fa7126e2ade1f9daaaeb8bd38b09e4 2024-11-12T14:49:28,803 INFO [StoreOpener-df409d6fa4df5d8fd91534d2d90eda60-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df409d6fa4df5d8fd91534d2d90eda60 columnFamilyName colfamily31 2024-11-12T14:49:28,803 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:49:28,803 DEBUG [StoreOpener-df409d6fa4df5d8fd91534d2d90eda60-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,804 INFO [StoreOpener-d3c298441d993e84c2d9e579914483db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region d3c298441d993e84c2d9e579914483db 2024-11-12T14:49:28,804 INFO [StoreOpener-df409d6fa4df5d8fd91534d2d90eda60-1 {}] regionserver.HStore(327): Store=df409d6fa4df5d8fd91534d2d90eda60/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,804 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for df409d6fa4df5d8fd91534d2d90eda60 2024-11-12T14:49:28,806 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/df409d6fa4df5d8fd91534d2d90eda60 2024-11-12T14:49:28,806 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 59fa7126e2ade1f9daaaeb8bd38b09e4 2024-11-12T14:49:28,806 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/df409d6fa4df5d8fd91534d2d90eda60 2024-11-12T14:49:28,808 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for df409d6fa4df5d8fd91534d2d90eda60 2024-11-12T14:49:28,808 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for df409d6fa4df5d8fd91534d2d90eda60 2024-11-12T14:49:28,809 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:49:28,810 INFO [StoreOpener-d3c298441d993e84c2d9e579914483db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3c298441d993e84c2d9e579914483db columnFamilyName colfamily11 2024-11-12T14:49:28,810 DEBUG [StoreOpener-d3c298441d993e84c2d9e579914483db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,810 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38303 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3945e1ca6f47,35473,1731422964757, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bb0382e4edc08a7c43f0571deb2374a4. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T14:49:28,811 INFO [StoreOpener-d3c298441d993e84c2d9e579914483db-1 {}] regionserver.HStore(327): Store=d3c298441d993e84c2d9e579914483db/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,812 INFO [StoreOpener-d3c298441d993e84c2d9e579914483db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region d3c298441d993e84c2d9e579914483db 2024-11-12T14:49:28,813 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for df409d6fa4df5d8fd91534d2d90eda60 2024-11-12T14:49:28,815 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=25 2024-11-12T14:49:28,815 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure efc23c8cc5ebe3a5c96ef5d199223832, server=3945e1ca6f47,45845,1731422964694 in 402 msec 2024-11-12T14:49:28,817 INFO [StoreOpener-d3c298441d993e84c2d9e579914483db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3c298441d993e84c2d9e579914483db columnFamilyName colfamily21 2024-11-12T14:49:28,817 DEBUG [StoreOpener-d3c298441d993e84c2d9e579914483db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,819 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=efc23c8cc5ebe3a5c96ef5d199223832, ASSIGN in 657 msec 2024-11-12T14:49:28,819 INFO [StoreOpener-d3c298441d993e84c2d9e579914483db-1 {}] regionserver.HStore(327): Store=d3c298441d993e84c2d9e579914483db/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,820 INFO [StoreOpener-d3c298441d993e84c2d9e579914483db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region d3c298441d993e84c2d9e579914483db 2024-11-12T14:49:28,822 INFO [StoreOpener-d3c298441d993e84c2d9e579914483db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3c298441d993e84c2d9e579914483db columnFamilyName colfamily31 2024-11-12T14:49:28,823 DEBUG [StoreOpener-d3c298441d993e84c2d9e579914483db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,825 INFO [StoreOpener-d3c298441d993e84c2d9e579914483db-1 {}] regionserver.HStore(327): Store=d3c298441d993e84c2d9e579914483db/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,825 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for d3c298441d993e84c2d9e579914483db 2024-11-12T14:49:28,827 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d3c298441d993e84c2d9e579914483db 2024-11-12T14:49:28,828 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/59fa7126e2ade1f9daaaeb8bd38b09e4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:49:28,828 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d3c298441d993e84c2d9e579914483db 2024-11-12T14:49:28,829 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=18 2024-11-12T14:49:28,829 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure d7c3d3ae3b942f0414dcbb110be8bc0b, server=3945e1ca6f47,37185,1731422964568 in 260 msec 2024-11-12T14:49:28,829 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 59fa7126e2ade1f9daaaeb8bd38b09e4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65439788, jitterRate=-0.02487117052078247}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:49:28,829 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 59fa7126e2ade1f9daaaeb8bd38b09e4 2024-11-12T14:49:28,829 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 59fa7126e2ade1f9daaaeb8bd38b09e4: Running coprocessor pre-open hook at 1731422968776Writing region info on filesystem at 1731422968776Initializing all the Stores at 1731422968778 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968778Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968779 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968779Cleaning up temporary data from old regions at 1731422968802 (+23 ms)Running coprocessor post-open hooks at 1731422968829 (+27 ms)Region opened successfully at 1731422968829 2024-11-12T14:49:28,830 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/df409d6fa4df5d8fd91534d2d90eda60/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:49:28,830 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for d3c298441d993e84c2d9e579914483db 2024-11-12T14:49:28,830 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for d3c298441d993e84c2d9e579914483db 2024-11-12T14:49:28,831 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened df409d6fa4df5d8fd91534d2d90eda60; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72287093, jitterRate=0.07716162502765656}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:49:28,831 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731422967229.59fa7126e2ade1f9daaaeb8bd38b09e4., pid=55, masterSystemTime=1731422968641 2024-11-12T14:49:28,831 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for df409d6fa4df5d8fd91534d2d90eda60 2024-11-12T14:49:28,831 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for df409d6fa4df5d8fd91534d2d90eda60: Running coprocessor pre-open hook at 1731422968784Writing region info on filesystem at 1731422968784Initializing all the Stores at 1731422968786 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968786Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968787 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968787Cleaning up temporary data from old regions at 1731422968808 (+21 ms)Running coprocessor post-open hooks at 1731422968831 (+23 ms)Region opened successfully at 1731422968831 2024-11-12T14:49:28,831 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:49:28,833 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731422967229.df409d6fa4df5d8fd91534d2d90eda60., pid=33, masterSystemTime=1731422968510 2024-11-12T14:49:28,834 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for d3c298441d993e84c2d9e579914483db 2024-11-12T14:49:28,835 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d7c3d3ae3b942f0414dcbb110be8bc0b, ASSIGN in 671 msec 2024-11-12T14:49:28,836 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731422967229.59fa7126e2ade1f9daaaeb8bd38b09e4. 2024-11-12T14:49:28,836 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731422967229.59fa7126e2ade1f9daaaeb8bd38b09e4. 2024-11-12T14:49:28,836 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731422967229.95e5de757651e0aa099998abaa3b598f. 2024-11-12T14:49:28,836 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 95e5de757651e0aa099998abaa3b598f, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731422967229.95e5de757651e0aa099998abaa3b598f.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-12T14:49:28,837 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 95e5de757651e0aa099998abaa3b598f 2024-11-12T14:49:28,837 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731422967229.95e5de757651e0aa099998abaa3b598f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:28,837 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 95e5de757651e0aa099998abaa3b598f 2024-11-12T14:49:28,838 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 95e5de757651e0aa099998abaa3b598f 2024-11-12T14:49:28,840 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731422967229.df409d6fa4df5d8fd91534d2d90eda60. 2024-11-12T14:49:28,840 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731422967229.df409d6fa4df5d8fd91534d2d90eda60. 2024-11-12T14:49:28,840 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731422967229.a2cc208a6afe25c72a2045fb03e0ecb1. 2024-11-12T14:49:28,841 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => a2cc208a6afe25c72a2045fb03e0ecb1, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731422967229.a2cc208a6afe25c72a2045fb03e0ecb1.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-12T14:49:28,841 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true a2cc208a6afe25c72a2045fb03e0ecb1 2024-11-12T14:49:28,841 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731422967229.a2cc208a6afe25c72a2045fb03e0ecb1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:28,842 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for a2cc208a6afe25c72a2045fb03e0ecb1 2024-11-12T14:49:28,842 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for a2cc208a6afe25c72a2045fb03e0ecb1 2024-11-12T14:49:28,842 INFO [StoreOpener-95e5de757651e0aa099998abaa3b598f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 95e5de757651e0aa099998abaa3b598f 2024-11-12T14:49:28,844 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d3c298441d993e84c2d9e579914483db/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:49:28,845 INFO [StoreOpener-95e5de757651e0aa099998abaa3b598f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 95e5de757651e0aa099998abaa3b598f columnFamilyName colfamily11 2024-11-12T14:49:28,845 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened d3c298441d993e84c2d9e579914483db; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69337190, jitterRate=0.03320464491844177}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:49:28,845 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d3c298441d993e84c2d9e579914483db 2024-11-12T14:49:28,845 DEBUG [StoreOpener-95e5de757651e0aa099998abaa3b598f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,845 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for d3c298441d993e84c2d9e579914483db: Running coprocessor pre-open hook at 1731422968792Writing region info on filesystem at 1731422968792Initializing all the Stores at 1731422968794 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968794Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968803 (+9 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968803Cleaning up temporary data from old regions at 1731422968830 (+27 ms)Running coprocessor post-open hooks at 1731422968845 (+15 ms)Region opened successfully at 1731422968845 2024-11-12T14:49:28,846 INFO [StoreOpener-95e5de757651e0aa099998abaa3b598f-1 {}] regionserver.HStore(327): Store=95e5de757651e0aa099998abaa3b598f/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,846 INFO [StoreOpener-95e5de757651e0aa099998abaa3b598f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 95e5de757651e0aa099998abaa3b598f 2024-11-12T14:49:28,847 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db., pid=49, masterSystemTime=1731422968613 2024-11-12T14:49:28,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=19 2024-11-12T14:49:28,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure bb0382e4edc08a7c43f0571deb2374a4, server=3945e1ca6f47,35473,1731422964757 in 319 msec 2024-11-12T14:49:28,849 INFO [StoreOpener-a2cc208a6afe25c72a2045fb03e0ecb1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region a2cc208a6afe25c72a2045fb03e0ecb1 2024-11-12T14:49:28,849 INFO [StoreOpener-95e5de757651e0aa099998abaa3b598f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 95e5de757651e0aa099998abaa3b598f columnFamilyName colfamily21 2024-11-12T14:49:28,849 DEBUG [StoreOpener-95e5de757651e0aa099998abaa3b598f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,851 INFO [StoreOpener-95e5de757651e0aa099998abaa3b598f-1 {}] regionserver.HStore(327): Store=95e5de757651e0aa099998abaa3b598f/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,852 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db. 2024-11-12T14:49:28,852 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db. 2024-11-12T14:49:28,852 INFO [StoreOpener-95e5de757651e0aa099998abaa3b598f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 95e5de757651e0aa099998abaa3b598f 2024-11-12T14:49:28,852 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38. 2024-11-12T14:49:28,852 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => e41036448a3677ed52f58f536690bf38, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-12T14:49:28,852 INFO [StoreOpener-a2cc208a6afe25c72a2045fb03e0ecb1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2cc208a6afe25c72a2045fb03e0ecb1 columnFamilyName colfamily11 2024-11-12T14:49:28,853 DEBUG [StoreOpener-a2cc208a6afe25c72a2045fb03e0ecb1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,853 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true e41036448a3677ed52f58f536690bf38 2024-11-12T14:49:28,853 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:28,853 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for e41036448a3677ed52f58f536690bf38 2024-11-12T14:49:28,853 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for e41036448a3677ed52f58f536690bf38 2024-11-12T14:49:28,854 INFO [StoreOpener-a2cc208a6afe25c72a2045fb03e0ecb1-1 {}] regionserver.HStore(327): Store=a2cc208a6afe25c72a2045fb03e0ecb1/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,855 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=bb0382e4edc08a7c43f0571deb2374a4, ASSIGN in 691 msec 2024-11-12T14:49:28,856 INFO [StoreOpener-95e5de757651e0aa099998abaa3b598f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 95e5de757651e0aa099998abaa3b598f columnFamilyName colfamily31 2024-11-12T14:49:28,856 DEBUG [StoreOpener-95e5de757651e0aa099998abaa3b598f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,856 INFO [StoreOpener-a2cc208a6afe25c72a2045fb03e0ecb1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region a2cc208a6afe25c72a2045fb03e0ecb1 2024-11-12T14:49:28,857 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=d3c298441d993e84c2d9e579914483db, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,37185,1731422964568 2024-11-12T14:49:28,857 INFO [StoreOpener-95e5de757651e0aa099998abaa3b598f-1 {}] regionserver.HStore(327): Store=95e5de757651e0aa099998abaa3b598f/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,857 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 95e5de757651e0aa099998abaa3b598f 2024-11-12T14:49:28,858 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/95e5de757651e0aa099998abaa3b598f 2024-11-12T14:49:28,860 INFO [StoreOpener-a2cc208a6afe25c72a2045fb03e0ecb1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2cc208a6afe25c72a2045fb03e0ecb1 columnFamilyName colfamily21 2024-11-12T14:49:28,860 DEBUG [StoreOpener-a2cc208a6afe25c72a2045fb03e0ecb1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,860 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/95e5de757651e0aa099998abaa3b598f 2024-11-12T14:49:28,861 INFO [StoreOpener-a2cc208a6afe25c72a2045fb03e0ecb1-1 {}] regionserver.HStore(327): Store=a2cc208a6afe25c72a2045fb03e0ecb1/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,861 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=df409d6fa4df5d8fd91534d2d90eda60, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,45845,1731422964694 2024-11-12T14:49:28,862 INFO [StoreOpener-a2cc208a6afe25c72a2045fb03e0ecb1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region a2cc208a6afe25c72a2045fb03e0ecb1 2024-11-12T14:49:28,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3c298441d993e84c2d9e579914483db, server=3945e1ca6f47,37185,1731422964568 because future has completed 2024-11-12T14:49:28,864 INFO [StoreOpener-a2cc208a6afe25c72a2045fb03e0ecb1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2cc208a6afe25c72a2045fb03e0ecb1 columnFamilyName colfamily31 2024-11-12T14:49:28,864 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 95e5de757651e0aa099998abaa3b598f 2024-11-12T14:49:28,865 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 95e5de757651e0aa099998abaa3b598f 2024-11-12T14:49:28,865 DEBUG [StoreOpener-a2cc208a6afe25c72a2045fb03e0ecb1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,866 INFO [StoreOpener-a2cc208a6afe25c72a2045fb03e0ecb1-1 {}] regionserver.HStore(327): Store=a2cc208a6afe25c72a2045fb03e0ecb1/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,866 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure df409d6fa4df5d8fd91534d2d90eda60, server=3945e1ca6f47,45845,1731422964694 because future has completed 2024-11-12T14:49:28,866 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for a2cc208a6afe25c72a2045fb03e0ecb1 2024-11-12T14:49:28,867 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:49:28,868 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a2cc208a6afe25c72a2045fb03e0ecb1 2024-11-12T14:49:28,869 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 95e5de757651e0aa099998abaa3b598f 2024-11-12T14:49:28,869 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a2cc208a6afe25c72a2045fb03e0ecb1 2024-11-12T14:49:28,872 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for a2cc208a6afe25c72a2045fb03e0ecb1 2024-11-12T14:49:28,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=9 2024-11-12T14:49:28,872 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for a2cc208a6afe25c72a2045fb03e0ecb1 2024-11-12T14:49:28,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure d3c298441d993e84c2d9e579914483db, server=3945e1ca6f47,37185,1731422964568 in 316 msec 2024-11-12T14:49:28,873 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:49:28,874 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/95e5de757651e0aa099998abaa3b598f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:49:28,875 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for a2cc208a6afe25c72a2045fb03e0ecb1 2024-11-12T14:49:28,875 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 95e5de757651e0aa099998abaa3b598f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59784691, jitterRate=-0.10913868248462677}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:49:28,875 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 95e5de757651e0aa099998abaa3b598f 2024-11-12T14:49:28,875 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 95e5de757651e0aa099998abaa3b598f: Running coprocessor pre-open hook at 1731422968838Writing region info on filesystem at 1731422968838Initializing all the Stores at 1731422968840 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968840Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968842 (+2 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968842Cleaning up temporary data from old regions at 1731422968865 (+23 ms)Running coprocessor post-open hooks at 1731422968875 (+10 ms)Region opened successfully at 1731422968875 2024-11-12T14:49:28,875 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d3c298441d993e84c2d9e579914483db, ASSIGN in 714 msec 2024-11-12T14:49:28,877 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731422967229.95e5de757651e0aa099998abaa3b598f., pid=47, masterSystemTime=1731422968641 2024-11-12T14:49:28,880 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731422967229.95e5de757651e0aa099998abaa3b598f. 2024-11-12T14:49:28,881 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731422967229.95e5de757651e0aa099998abaa3b598f. 2024-11-12T14:49:28,881 INFO [StoreOpener-e41036448a3677ed52f58f536690bf38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region e41036448a3677ed52f58f536690bf38 2024-11-12T14:49:28,882 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a2cc208a6afe25c72a2045fb03e0ecb1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:49:28,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=13 2024-11-12T14:49:28,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure df409d6fa4df5d8fd91534d2d90eda60, server=3945e1ca6f47,45845,1731422964694 in 511 msec 2024-11-12T14:49:28,882 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731422967229.8b5e68238ca126132dbf55f0fbf38047. 2024-11-12T14:49:28,883 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 8b5e68238ca126132dbf55f0fbf38047, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731422967229.8b5e68238ca126132dbf55f0fbf38047.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-12T14:49:28,883 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened a2cc208a6afe25c72a2045fb03e0ecb1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58723006, jitterRate=-0.12495902180671692}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:49:28,883 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a2cc208a6afe25c72a2045fb03e0ecb1 2024-11-12T14:49:28,884 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for a2cc208a6afe25c72a2045fb03e0ecb1: Running coprocessor pre-open hook at 1731422968842Writing region info on filesystem at 1731422968842Initializing all the Stores at 1731422968843 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968843Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968849 (+6 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968849Cleaning up temporary data from old regions at 1731422968872 (+23 ms)Running coprocessor post-open hooks at 1731422968883 (+11 ms)Region opened successfully at 1731422968883 2024-11-12T14:49:28,884 INFO [StoreOpener-e41036448a3677ed52f58f536690bf38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e41036448a3677ed52f58f536690bf38 columnFamilyName colfamily11 2024-11-12T14:49:28,884 DEBUG [StoreOpener-e41036448a3677ed52f58f536690bf38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,885 INFO [StoreOpener-e41036448a3677ed52f58f536690bf38-1 {}] regionserver.HStore(327): Store=e41036448a3677ed52f58f536690bf38/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,885 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731422967229.a2cc208a6afe25c72a2045fb03e0ecb1., pid=36, masterSystemTime=1731422968510 2024-11-12T14:49:28,885 INFO [StoreOpener-e41036448a3677ed52f58f536690bf38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region e41036448a3677ed52f58f536690bf38 2024-11-12T14:49:28,885 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 8b5e68238ca126132dbf55f0fbf38047 2024-11-12T14:49:28,885 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731422967229.8b5e68238ca126132dbf55f0fbf38047.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:28,885 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 8b5e68238ca126132dbf55f0fbf38047 2024-11-12T14:49:28,887 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 8b5e68238ca126132dbf55f0fbf38047 2024-11-12T14:49:28,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=df409d6fa4df5d8fd91534d2d90eda60, ASSIGN in 725 msec 2024-11-12T14:49:28,887 INFO [StoreOpener-e41036448a3677ed52f58f536690bf38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e41036448a3677ed52f58f536690bf38 columnFamilyName colfamily21 2024-11-12T14:49:28,887 DEBUG [StoreOpener-e41036448a3677ed52f58f536690bf38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,888 INFO [StoreOpener-e41036448a3677ed52f58f536690bf38-1 {}] regionserver.HStore(327): Store=e41036448a3677ed52f58f536690bf38/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,888 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731422967229.a2cc208a6afe25c72a2045fb03e0ecb1. 2024-11-12T14:49:28,889 INFO [StoreOpener-e41036448a3677ed52f58f536690bf38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region e41036448a3677ed52f58f536690bf38 2024-11-12T14:49:28,889 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731422967229.a2cc208a6afe25c72a2045fb03e0ecb1. 2024-11-12T14:49:28,889 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731422967229.cf0006d9b8777107f458b7008fc42ba9. 2024-11-12T14:49:28,889 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=95e5de757651e0aa099998abaa3b598f, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,35473,1731422964757 2024-11-12T14:49:28,889 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => cf0006d9b8777107f458b7008fc42ba9, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731422967229.cf0006d9b8777107f458b7008fc42ba9.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-12T14:49:28,890 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true cf0006d9b8777107f458b7008fc42ba9 2024-11-12T14:49:28,890 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731422967229.cf0006d9b8777107f458b7008fc42ba9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:28,890 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for cf0006d9b8777107f458b7008fc42ba9 2024-11-12T14:49:28,890 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for cf0006d9b8777107f458b7008fc42ba9 2024-11-12T14:49:28,893 INFO [StoreOpener-e41036448a3677ed52f58f536690bf38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e41036448a3677ed52f58f536690bf38 columnFamilyName colfamily31 2024-11-12T14:49:28,893 DEBUG [StoreOpener-e41036448a3677ed52f58f536690bf38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,893 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 95e5de757651e0aa099998abaa3b598f, server=3945e1ca6f47,35473,1731422964757 because future has completed 2024-11-12T14:49:28,894 INFO [StoreOpener-cf0006d9b8777107f458b7008fc42ba9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region cf0006d9b8777107f458b7008fc42ba9 2024-11-12T14:49:28,894 INFO [StoreOpener-e41036448a3677ed52f58f536690bf38-1 {}] regionserver.HStore(327): Store=e41036448a3677ed52f58f536690bf38/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,894 INFO [StoreOpener-8b5e68238ca126132dbf55f0fbf38047-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 8b5e68238ca126132dbf55f0fbf38047 2024-11-12T14:49:28,894 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for e41036448a3677ed52f58f536690bf38 2024-11-12T14:49:28,895 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=a2cc208a6afe25c72a2045fb03e0ecb1, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,45845,1731422964694 2024-11-12T14:49:28,895 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e41036448a3677ed52f58f536690bf38 2024-11-12T14:49:28,896 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e41036448a3677ed52f58f536690bf38 2024-11-12T14:49:28,896 INFO [StoreOpener-cf0006d9b8777107f458b7008fc42ba9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf0006d9b8777107f458b7008fc42ba9 columnFamilyName colfamily11 2024-11-12T14:49:28,896 DEBUG [StoreOpener-cf0006d9b8777107f458b7008fc42ba9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,898 INFO [StoreOpener-cf0006d9b8777107f458b7008fc42ba9-1 {}] regionserver.HStore(327): Store=cf0006d9b8777107f458b7008fc42ba9/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,898 INFO [StoreOpener-cf0006d9b8777107f458b7008fc42ba9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region cf0006d9b8777107f458b7008fc42ba9 2024-11-12T14:49:28,899 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for e41036448a3677ed52f58f536690bf38 2024-11-12T14:49:28,899 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for e41036448a3677ed52f58f536690bf38 2024-11-12T14:49:28,900 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure a2cc208a6afe25c72a2045fb03e0ecb1, server=3945e1ca6f47,45845,1731422964694 because future has completed 2024-11-12T14:49:28,900 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:49:28,901 INFO [StoreOpener-cf0006d9b8777107f458b7008fc42ba9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf0006d9b8777107f458b7008fc42ba9 columnFamilyName colfamily21 2024-11-12T14:49:28,901 DEBUG [StoreOpener-cf0006d9b8777107f458b7008fc42ba9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,902 INFO [StoreOpener-cf0006d9b8777107f458b7008fc42ba9-1 {}] regionserver.HStore(327): Store=cf0006d9b8777107f458b7008fc42ba9/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,902 INFO [StoreOpener-cf0006d9b8777107f458b7008fc42ba9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region cf0006d9b8777107f458b7008fc42ba9 2024-11-12T14:49:28,902 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for e41036448a3677ed52f58f536690bf38 2024-11-12T14:49:28,905 INFO [StoreOpener-cf0006d9b8777107f458b7008fc42ba9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf0006d9b8777107f458b7008fc42ba9 columnFamilyName colfamily31 2024-11-12T14:49:28,905 INFO [StoreOpener-8b5e68238ca126132dbf55f0fbf38047-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b5e68238ca126132dbf55f0fbf38047 columnFamilyName colfamily11 2024-11-12T14:49:28,906 DEBUG [StoreOpener-8b5e68238ca126132dbf55f0fbf38047-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,906 DEBUG [StoreOpener-cf0006d9b8777107f458b7008fc42ba9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,906 INFO [StoreOpener-8b5e68238ca126132dbf55f0fbf38047-1 {}] regionserver.HStore(327): Store=8b5e68238ca126132dbf55f0fbf38047/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,907 INFO [StoreOpener-8b5e68238ca126132dbf55f0fbf38047-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 8b5e68238ca126132dbf55f0fbf38047 2024-11-12T14:49:28,906 INFO [StoreOpener-cf0006d9b8777107f458b7008fc42ba9-1 {}] regionserver.HStore(327): Store=cf0006d9b8777107f458b7008fc42ba9/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,907 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for cf0006d9b8777107f458b7008fc42ba9 2024-11-12T14:49:28,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=29 2024-11-12T14:49:28,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 95e5de757651e0aa099998abaa3b598f, server=3945e1ca6f47,35473,1731422964757 in 368 msec 2024-11-12T14:49:28,909 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/cf0006d9b8777107f458b7008fc42ba9 2024-11-12T14:49:28,909 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/cf0006d9b8777107f458b7008fc42ba9 2024-11-12T14:49:28,910 INFO [StoreOpener-8b5e68238ca126132dbf55f0fbf38047-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b5e68238ca126132dbf55f0fbf38047 columnFamilyName colfamily21 2024-11-12T14:49:28,910 DEBUG [StoreOpener-8b5e68238ca126132dbf55f0fbf38047-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,911 INFO [StoreOpener-8b5e68238ca126132dbf55f0fbf38047-1 {}] regionserver.HStore(327): Store=8b5e68238ca126132dbf55f0fbf38047/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,911 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for cf0006d9b8777107f458b7008fc42ba9 2024-11-12T14:49:28,911 INFO [StoreOpener-8b5e68238ca126132dbf55f0fbf38047-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 8b5e68238ca126132dbf55f0fbf38047 2024-11-12T14:49:28,911 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for cf0006d9b8777107f458b7008fc42ba9 2024-11-12T14:49:28,911 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e41036448a3677ed52f58f536690bf38/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:49:28,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=95e5de757651e0aa099998abaa3b598f, ASSIGN in 750 msec 2024-11-12T14:49:28,912 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened e41036448a3677ed52f58f536690bf38; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59310974, jitterRate=-0.1161976158618927}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:49:28,912 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:49:28,912 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e41036448a3677ed52f58f536690bf38 2024-11-12T14:49:28,912 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for e41036448a3677ed52f58f536690bf38: Running coprocessor pre-open hook at 1731422968853Writing region info on filesystem at 1731422968853Initializing all the Stores at 1731422968856 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968856Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968865 (+9 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968865Cleaning up temporary data from old regions at 1731422968899 (+34 ms)Running coprocessor post-open hooks at 1731422968912 (+13 ms)Region opened successfully at 1731422968912 2024-11-12T14:49:28,914 INFO [StoreOpener-8b5e68238ca126132dbf55f0fbf38047-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b5e68238ca126132dbf55f0fbf38047 columnFamilyName colfamily31 2024-11-12T14:49:28,914 DEBUG [StoreOpener-8b5e68238ca126132dbf55f0fbf38047-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,914 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for cf0006d9b8777107f458b7008fc42ba9 2024-11-12T14:49:28,915 INFO [StoreOpener-8b5e68238ca126132dbf55f0fbf38047-1 {}] regionserver.HStore(327): Store=8b5e68238ca126132dbf55f0fbf38047/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,915 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 8b5e68238ca126132dbf55f0fbf38047 2024-11-12T14:49:28,916 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38., pid=53, masterSystemTime=1731422968613 2024-11-12T14:49:28,916 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8b5e68238ca126132dbf55f0fbf38047 2024-11-12T14:49:28,917 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8b5e68238ca126132dbf55f0fbf38047 2024-11-12T14:49:28,920 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 8b5e68238ca126132dbf55f0fbf38047 2024-11-12T14:49:28,920 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38. 2024-11-12T14:49:28,920 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 8b5e68238ca126132dbf55f0fbf38047 2024-11-12T14:49:28,920 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/cf0006d9b8777107f458b7008fc42ba9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:49:28,920 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38. 2024-11-12T14:49:28,920 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06. 2024-11-12T14:49:28,920 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 92f3e9baa717d57147319661f6f00e06, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-12T14:49:28,920 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:49:28,921 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened cf0006d9b8777107f458b7008fc42ba9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73631480, jitterRate=0.09719455242156982}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:49:28,921 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cf0006d9b8777107f458b7008fc42ba9 2024-11-12T14:49:28,921 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 92f3e9baa717d57147319661f6f00e06 2024-11-12T14:49:28,921 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:28,921 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for cf0006d9b8777107f458b7008fc42ba9: Running coprocessor pre-open hook at 1731422968890Writing region info on filesystem at 1731422968890Initializing all the Stores at 1731422968893 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968893Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968894 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968894Cleaning up temporary data from old regions at 1731422968911 (+17 ms)Running coprocessor post-open hooks at 1731422968921 (+10 ms)Region opened successfully at 1731422968921 2024-11-12T14:49:28,921 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 92f3e9baa717d57147319661f6f00e06 2024-11-12T14:49:28,922 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 92f3e9baa717d57147319661f6f00e06 2024-11-12T14:49:28,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=22 2024-11-12T14:49:28,922 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731422967229.cf0006d9b8777107f458b7008fc42ba9., pid=38, masterSystemTime=1731422968510 2024-11-12T14:49:28,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure a2cc208a6afe25c72a2045fb03e0ecb1, server=3945e1ca6f47,45845,1731422964694 in 528 msec 2024-11-12T14:49:28,923 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 8b5e68238ca126132dbf55f0fbf38047 2024-11-12T14:49:28,925 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731422967229.cf0006d9b8777107f458b7008fc42ba9. 2024-11-12T14:49:28,926 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731422967229.cf0006d9b8777107f458b7008fc42ba9. 2024-11-12T14:49:28,926 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731422967229.b419710b4e7b088b11da63dff9b7760e. 2024-11-12T14:49:28,926 INFO [StoreOpener-92f3e9baa717d57147319661f6f00e06-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 92f3e9baa717d57147319661f6f00e06 2024-11-12T14:49:28,926 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => b419710b4e7b088b11da63dff9b7760e, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731422967229.b419710b4e7b088b11da63dff9b7760e.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-12T14:49:28,927 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8b5e68238ca126132dbf55f0fbf38047/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:49:28,927 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true b419710b4e7b088b11da63dff9b7760e 2024-11-12T14:49:28,927 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=a2cc208a6afe25c72a2045fb03e0ecb1, ASSIGN in 764 msec 2024-11-12T14:49:28,927 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731422967229.b419710b4e7b088b11da63dff9b7760e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:28,927 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for b419710b4e7b088b11da63dff9b7760e 2024-11-12T14:49:28,927 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 8b5e68238ca126132dbf55f0fbf38047; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61107652, jitterRate=-0.08942502737045288}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:49:28,927 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for b419710b4e7b088b11da63dff9b7760e 2024-11-12T14:49:28,927 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8b5e68238ca126132dbf55f0fbf38047 2024-11-12T14:49:28,928 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 8b5e68238ca126132dbf55f0fbf38047: Running coprocessor pre-open hook at 1731422968887Writing region info on filesystem at 1731422968887Initializing all the Stores at 1731422968889 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968889Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968893 (+4 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968893Cleaning up temporary data from old regions at 1731422968920 (+27 ms)Running coprocessor post-open hooks at 1731422968927 (+7 ms)Region opened successfully at 1731422968928 (+1 ms) 2024-11-12T14:49:28,930 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731422967229.8b5e68238ca126132dbf55f0fbf38047., pid=42, masterSystemTime=1731422968641 2024-11-12T14:49:28,931 INFO [StoreOpener-92f3e9baa717d57147319661f6f00e06-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92f3e9baa717d57147319661f6f00e06 columnFamilyName colfamily11 2024-11-12T14:49:28,931 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=cf0006d9b8777107f458b7008fc42ba9, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,45845,1731422964694 2024-11-12T14:49:28,931 DEBUG [StoreOpener-92f3e9baa717d57147319661f6f00e06-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,932 INFO [StoreOpener-92f3e9baa717d57147319661f6f00e06-1 {}] regionserver.HStore(327): Store=92f3e9baa717d57147319661f6f00e06/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,932 INFO [StoreOpener-92f3e9baa717d57147319661f6f00e06-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 92f3e9baa717d57147319661f6f00e06 2024-11-12T14:49:28,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure cf0006d9b8777107f458b7008fc42ba9, server=3945e1ca6f47,45845,1731422964694 because future has completed 2024-11-12T14:49:28,935 INFO [StoreOpener-92f3e9baa717d57147319661f6f00e06-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92f3e9baa717d57147319661f6f00e06 columnFamilyName colfamily21 2024-11-12T14:49:28,935 DEBUG [StoreOpener-92f3e9baa717d57147319661f6f00e06-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,936 INFO [StoreOpener-92f3e9baa717d57147319661f6f00e06-1 {}] regionserver.HStore(327): Store=92f3e9baa717d57147319661f6f00e06/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,936 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731422967229.8b5e68238ca126132dbf55f0fbf38047. 2024-11-12T14:49:28,936 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731422967229.8b5e68238ca126132dbf55f0fbf38047. 2024-11-12T14:49:28,936 INFO [StoreOpener-92f3e9baa717d57147319661f6f00e06-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 92f3e9baa717d57147319661f6f00e06 2024-11-12T14:49:28,936 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731422967229.f65e71cd750425cce9713e30a75c6002. 2024-11-12T14:49:28,936 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38303 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3945e1ca6f47,45845,1731422964694, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=cf0006d9b8777107f458b7008fc42ba9. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T14:49:28,936 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => f65e71cd750425cce9713e30a75c6002, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731422967229.f65e71cd750425cce9713e30a75c6002.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-12T14:49:28,937 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true f65e71cd750425cce9713e30a75c6002 2024-11-12T14:49:28,937 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=e41036448a3677ed52f58f536690bf38, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,37185,1731422964568 2024-11-12T14:49:28,937 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731422967229.f65e71cd750425cce9713e30a75c6002.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:28,937 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for f65e71cd750425cce9713e30a75c6002 2024-11-12T14:49:28,937 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for f65e71cd750425cce9713e30a75c6002 2024-11-12T14:49:28,937 INFO [StoreOpener-b419710b4e7b088b11da63dff9b7760e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region b419710b4e7b088b11da63dff9b7760e 2024-11-12T14:49:28,939 INFO [StoreOpener-92f3e9baa717d57147319661f6f00e06-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92f3e9baa717d57147319661f6f00e06 columnFamilyName colfamily31 2024-11-12T14:49:28,939 DEBUG [StoreOpener-92f3e9baa717d57147319661f6f00e06-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,940 INFO [StoreOpener-b419710b4e7b088b11da63dff9b7760e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b419710b4e7b088b11da63dff9b7760e columnFamilyName colfamily11 2024-11-12T14:49:28,940 DEBUG [StoreOpener-b419710b4e7b088b11da63dff9b7760e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,941 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure e41036448a3677ed52f58f536690bf38, server=3945e1ca6f47,37185,1731422964568 because future has completed 2024-11-12T14:49:28,941 INFO [StoreOpener-92f3e9baa717d57147319661f6f00e06-1 {}] regionserver.HStore(327): Store=92f3e9baa717d57147319661f6f00e06/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,941 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 92f3e9baa717d57147319661f6f00e06 2024-11-12T14:49:28,941 INFO [StoreOpener-b419710b4e7b088b11da63dff9b7760e-1 {}] regionserver.HStore(327): Store=b419710b4e7b088b11da63dff9b7760e/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,941 INFO [StoreOpener-b419710b4e7b088b11da63dff9b7760e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region b419710b4e7b088b11da63dff9b7760e 2024-11-12T14:49:28,942 INFO [StoreOpener-f65e71cd750425cce9713e30a75c6002-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region f65e71cd750425cce9713e30a75c6002 2024-11-12T14:49:28,943 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/92f3e9baa717d57147319661f6f00e06 2024-11-12T14:49:28,943 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=8b5e68238ca126132dbf55f0fbf38047, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,35473,1731422964757 2024-11-12T14:49:28,943 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/92f3e9baa717d57147319661f6f00e06 2024-11-12T14:49:28,944 INFO [StoreOpener-b419710b4e7b088b11da63dff9b7760e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b419710b4e7b088b11da63dff9b7760e columnFamilyName colfamily21 2024-11-12T14:49:28,944 INFO [StoreOpener-f65e71cd750425cce9713e30a75c6002-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f65e71cd750425cce9713e30a75c6002 columnFamilyName colfamily11 2024-11-12T14:49:28,944 DEBUG [StoreOpener-f65e71cd750425cce9713e30a75c6002-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,944 DEBUG [StoreOpener-b419710b4e7b088b11da63dff9b7760e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,945 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 92f3e9baa717d57147319661f6f00e06 2024-11-12T14:49:28,945 INFO [StoreOpener-b419710b4e7b088b11da63dff9b7760e-1 {}] regionserver.HStore(327): Store=b419710b4e7b088b11da63dff9b7760e/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,945 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 92f3e9baa717d57147319661f6f00e06 2024-11-12T14:49:28,945 INFO [StoreOpener-b419710b4e7b088b11da63dff9b7760e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region b419710b4e7b088b11da63dff9b7760e 2024-11-12T14:49:28,946 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:49:28,946 INFO [StoreOpener-f65e71cd750425cce9713e30a75c6002-1 {}] regionserver.HStore(327): Store=f65e71cd750425cce9713e30a75c6002/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,946 INFO [StoreOpener-f65e71cd750425cce9713e30a75c6002-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region f65e71cd750425cce9713e30a75c6002 2024-11-12T14:49:28,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b5e68238ca126132dbf55f0fbf38047, server=3945e1ca6f47,35473,1731422964757 because future has completed 2024-11-12T14:49:28,948 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 92f3e9baa717d57147319661f6f00e06 2024-11-12T14:49:28,948 INFO [StoreOpener-b419710b4e7b088b11da63dff9b7760e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b419710b4e7b088b11da63dff9b7760e columnFamilyName colfamily31 2024-11-12T14:49:28,948 DEBUG [StoreOpener-b419710b4e7b088b11da63dff9b7760e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,949 INFO [StoreOpener-b419710b4e7b088b11da63dff9b7760e-1 {}] regionserver.HStore(327): Store=b419710b4e7b088b11da63dff9b7760e/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,949 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for b419710b4e7b088b11da63dff9b7760e 2024-11-12T14:49:28,949 INFO [StoreOpener-f65e71cd750425cce9713e30a75c6002-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f65e71cd750425cce9713e30a75c6002 columnFamilyName colfamily21 2024-11-12T14:49:28,949 DEBUG [StoreOpener-f65e71cd750425cce9713e30a75c6002-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,950 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b419710b4e7b088b11da63dff9b7760e 2024-11-12T14:49:28,950 INFO [StoreOpener-f65e71cd750425cce9713e30a75c6002-1 {}] regionserver.HStore(327): Store=f65e71cd750425cce9713e30a75c6002/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,950 INFO [StoreOpener-f65e71cd750425cce9713e30a75c6002-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region f65e71cd750425cce9713e30a75c6002 2024-11-12T14:49:28,950 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b419710b4e7b088b11da63dff9b7760e 2024-11-12T14:49:28,952 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for b419710b4e7b088b11da63dff9b7760e 2024-11-12T14:49:28,952 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for b419710b4e7b088b11da63dff9b7760e 2024-11-12T14:49:28,953 INFO [StoreOpener-f65e71cd750425cce9713e30a75c6002-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f65e71cd750425cce9713e30a75c6002 columnFamilyName colfamily31 2024-11-12T14:49:28,953 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:49:28,953 DEBUG [StoreOpener-f65e71cd750425cce9713e30a75c6002-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,954 INFO [StoreOpener-f65e71cd750425cce9713e30a75c6002-1 {}] regionserver.HStore(327): Store=f65e71cd750425cce9713e30a75c6002/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,954 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/92f3e9baa717d57147319661f6f00e06/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:49:28,954 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for f65e71cd750425cce9713e30a75c6002 2024-11-12T14:49:28,955 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for b419710b4e7b088b11da63dff9b7760e 2024-11-12T14:49:28,955 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 92f3e9baa717d57147319661f6f00e06; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60475643, jitterRate=-0.09884269535541534}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:49:28,955 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 92f3e9baa717d57147319661f6f00e06 2024-11-12T14:49:28,955 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 92f3e9baa717d57147319661f6f00e06: Running coprocessor pre-open hook at 1731422968922Writing region info on filesystem at 1731422968922Initializing all the Stores at 1731422968924 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968924Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968925 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968925Cleaning up temporary data from old regions at 1731422968945 (+20 ms)Running coprocessor post-open hooks at 1731422968955 (+10 ms)Region opened successfully at 1731422968955 2024-11-12T14:49:28,955 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f65e71cd750425cce9713e30a75c6002 2024-11-12T14:49:28,956 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f65e71cd750425cce9713e30a75c6002 2024-11-12T14:49:28,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=30 2024-11-12T14:49:28,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure e41036448a3677ed52f58f536690bf38, server=3945e1ca6f47,37185,1731422964568 in 357 msec 2024-11-12T14:49:28,957 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06., pid=40, masterSystemTime=1731422968613 2024-11-12T14:49:28,958 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for f65e71cd750425cce9713e30a75c6002 2024-11-12T14:49:28,958 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for f65e71cd750425cce9713e30a75c6002 2024-11-12T14:49:28,959 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:49:28,960 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b419710b4e7b088b11da63dff9b7760e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:49:28,961 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for f65e71cd750425cce9713e30a75c6002 2024-11-12T14:49:28,961 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06. 2024-11-12T14:49:28,961 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened b419710b4e7b088b11da63dff9b7760e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69839344, jitterRate=0.04068732261657715}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:49:28,961 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b419710b4e7b088b11da63dff9b7760e 2024-11-12T14:49:28,961 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06. 2024-11-12T14:49:28,961 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for b419710b4e7b088b11da63dff9b7760e: Running coprocessor pre-open hook at 1731422968928Writing region info on filesystem at 1731422968928Initializing all the Stores at 1731422968930 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968930Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968937 (+7 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968937Cleaning up temporary data from old regions at 1731422968952 (+15 ms)Running coprocessor post-open hooks at 1731422968961 (+9 ms)Region opened successfully at 1731422968961 2024-11-12T14:49:28,961 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b. 2024-11-12T14:49:28,961 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 9a6f7a226796c8ff7e21a40d463cd27b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-12T14:49:28,962 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 9a6f7a226796c8ff7e21a40d463cd27b 2024-11-12T14:49:28,962 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:28,962 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 9a6f7a226796c8ff7e21a40d463cd27b 2024-11-12T14:49:28,962 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 9a6f7a226796c8ff7e21a40d463cd27b 2024-11-12T14:49:28,964 INFO [StoreOpener-9a6f7a226796c8ff7e21a40d463cd27b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 9a6f7a226796c8ff7e21a40d463cd27b 2024-11-12T14:49:28,965 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731422967229.b419710b4e7b088b11da63dff9b7760e., pid=35, masterSystemTime=1731422968510 2024-11-12T14:49:28,965 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f65e71cd750425cce9713e30a75c6002/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:49:28,966 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e41036448a3677ed52f58f536690bf38, ASSIGN in 799 msec 2024-11-12T14:49:28,966 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened f65e71cd750425cce9713e30a75c6002; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73062962, jitterRate=0.08872297406196594}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:49:28,966 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f65e71cd750425cce9713e30a75c6002 2024-11-12T14:49:28,967 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for f65e71cd750425cce9713e30a75c6002: Running coprocessor pre-open hook at 1731422968937Writing region info on filesystem at 1731422968937Initializing all the Stores at 1731422968939 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968939Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968941 (+2 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968941Cleaning up temporary data from old regions at 1731422968958 (+17 ms)Running coprocessor post-open hooks at 1731422968966 (+8 ms)Region opened successfully at 1731422968967 (+1 ms) 2024-11-12T14:49:28,968 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731422967229.f65e71cd750425cce9713e30a75c6002., pid=43, masterSystemTime=1731422968641 2024-11-12T14:49:28,968 INFO [StoreOpener-9a6f7a226796c8ff7e21a40d463cd27b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a6f7a226796c8ff7e21a40d463cd27b columnFamilyName colfamily11 2024-11-12T14:49:28,968 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=92f3e9baa717d57147319661f6f00e06, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,37185,1731422964568 2024-11-12T14:49:28,968 DEBUG [StoreOpener-9a6f7a226796c8ff7e21a40d463cd27b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,969 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731422967229.b419710b4e7b088b11da63dff9b7760e. 2024-11-12T14:49:28,969 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731422967229.b419710b4e7b088b11da63dff9b7760e. 2024-11-12T14:49:28,969 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419. 2024-11-12T14:49:28,970 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => f943ce043844f379c1b7ecbf76b5a419, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-12T14:49:28,971 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true f943ce043844f379c1b7ecbf76b5a419 2024-11-12T14:49:28,971 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:28,971 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for f943ce043844f379c1b7ecbf76b5a419 2024-11-12T14:49:28,971 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for f943ce043844f379c1b7ecbf76b5a419 2024-11-12T14:49:28,972 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92f3e9baa717d57147319661f6f00e06, server=3945e1ca6f47,37185,1731422964568 because future has completed 2024-11-12T14:49:28,972 INFO [StoreOpener-9a6f7a226796c8ff7e21a40d463cd27b-1 {}] regionserver.HStore(327): Store=9a6f7a226796c8ff7e21a40d463cd27b/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,972 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731422967229.f65e71cd750425cce9713e30a75c6002. 2024-11-12T14:49:28,972 INFO [StoreOpener-9a6f7a226796c8ff7e21a40d463cd27b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 9a6f7a226796c8ff7e21a40d463cd27b 2024-11-12T14:49:28,972 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731422967229.f65e71cd750425cce9713e30a75c6002. 2024-11-12T14:49:28,973 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731422967229.ac225c32dbf496a52e546c7e0b1a05af. 2024-11-12T14:49:28,973 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => ac225c32dbf496a52e546c7e0b1a05af, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731422967229.ac225c32dbf496a52e546c7e0b1a05af.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-12T14:49:28,973 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true ac225c32dbf496a52e546c7e0b1a05af 2024-11-12T14:49:28,973 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731422967229.ac225c32dbf496a52e546c7e0b1a05af.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:28,974 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for ac225c32dbf496a52e546c7e0b1a05af 2024-11-12T14:49:28,974 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=f65e71cd750425cce9713e30a75c6002, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,35473,1731422964757 2024-11-12T14:49:28,974 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for ac225c32dbf496a52e546c7e0b1a05af 2024-11-12T14:49:28,974 INFO [StoreOpener-f943ce043844f379c1b7ecbf76b5a419-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region f943ce043844f379c1b7ecbf76b5a419 2024-11-12T14:49:28,975 INFO [StoreOpener-9a6f7a226796c8ff7e21a40d463cd27b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a6f7a226796c8ff7e21a40d463cd27b columnFamilyName colfamily21 2024-11-12T14:49:28,975 DEBUG [StoreOpener-9a6f7a226796c8ff7e21a40d463cd27b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,976 INFO [StoreOpener-9a6f7a226796c8ff7e21a40d463cd27b-1 {}] regionserver.HStore(327): Store=9a6f7a226796c8ff7e21a40d463cd27b/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,976 INFO [StoreOpener-9a6f7a226796c8ff7e21a40d463cd27b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 9a6f7a226796c8ff7e21a40d463cd27b 2024-11-12T14:49:28,978 INFO [StoreOpener-f943ce043844f379c1b7ecbf76b5a419-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f943ce043844f379c1b7ecbf76b5a419 columnFamilyName colfamily11 2024-11-12T14:49:28,978 DEBUG [StoreOpener-f943ce043844f379c1b7ecbf76b5a419-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,978 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure f65e71cd750425cce9713e30a75c6002, server=3945e1ca6f47,35473,1731422964757 because future has completed 2024-11-12T14:49:28,979 INFO [StoreOpener-f943ce043844f379c1b7ecbf76b5a419-1 {}] regionserver.HStore(327): Store=f943ce043844f379c1b7ecbf76b5a419/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,979 INFO [StoreOpener-9a6f7a226796c8ff7e21a40d463cd27b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a6f7a226796c8ff7e21a40d463cd27b columnFamilyName colfamily31 2024-11-12T14:49:28,979 DEBUG [StoreOpener-9a6f7a226796c8ff7e21a40d463cd27b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,979 INFO [StoreOpener-f943ce043844f379c1b7ecbf76b5a419-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region f943ce043844f379c1b7ecbf76b5a419 2024-11-12T14:49:28,980 INFO [StoreOpener-9a6f7a226796c8ff7e21a40d463cd27b-1 {}] regionserver.HStore(327): Store=9a6f7a226796c8ff7e21a40d463cd27b/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,980 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 9a6f7a226796c8ff7e21a40d463cd27b 2024-11-12T14:49:28,981 INFO [StoreOpener-f943ce043844f379c1b7ecbf76b5a419-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f943ce043844f379c1b7ecbf76b5a419 columnFamilyName colfamily21 2024-11-12T14:49:28,981 DEBUG [StoreOpener-f943ce043844f379c1b7ecbf76b5a419-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,981 INFO [StoreOpener-f943ce043844f379c1b7ecbf76b5a419-1 {}] regionserver.HStore(327): Store=f943ce043844f379c1b7ecbf76b5a419/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,982 INFO [StoreOpener-f943ce043844f379c1b7ecbf76b5a419-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region f943ce043844f379c1b7ecbf76b5a419 2024-11-12T14:49:28,982 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9a6f7a226796c8ff7e21a40d463cd27b 2024-11-12T14:49:28,982 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9a6f7a226796c8ff7e21a40d463cd27b 2024-11-12T14:49:28,983 INFO [StoreOpener-f943ce043844f379c1b7ecbf76b5a419-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f943ce043844f379c1b7ecbf76b5a419 columnFamilyName colfamily31 2024-11-12T14:49:28,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=14 2024-11-12T14:49:28,984 DEBUG [StoreOpener-f943ce043844f379c1b7ecbf76b5a419-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:28,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 92f3e9baa717d57147319661f6f00e06, server=3945e1ca6f47,37185,1731422964568 in 517 msec 2024-11-12T14:49:28,984 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 9a6f7a226796c8ff7e21a40d463cd27b 2024-11-12T14:49:28,984 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 9a6f7a226796c8ff7e21a40d463cd27b 2024-11-12T14:49:28,984 INFO [StoreOpener-f943ce043844f379c1b7ecbf76b5a419-1 {}] regionserver.HStore(327): Store=f943ce043844f379c1b7ecbf76b5a419/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:28,984 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:49:28,985 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for f943ce043844f379c1b7ecbf76b5a419 2024-11-12T14:49:28,986 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 9a6f7a226796c8ff7e21a40d463cd27b 2024-11-12T14:49:28,987 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f943ce043844f379c1b7ecbf76b5a419 2024-11-12T14:49:28,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=92f3e9baa717d57147319661f6f00e06, ASSIGN in 826 msec 2024-11-12T14:49:28,987 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f943ce043844f379c1b7ecbf76b5a419 2024-11-12T14:49:28,990 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for f943ce043844f379c1b7ecbf76b5a419 2024-11-12T14:49:28,990 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for f943ce043844f379c1b7ecbf76b5a419 2024-11-12T14:49:28,990 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9a6f7a226796c8ff7e21a40d463cd27b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:49:28,991 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 9a6f7a226796c8ff7e21a40d463cd27b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60158768, jitterRate=-0.10356450080871582}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:49:28,991 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:49:28,991 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9a6f7a226796c8ff7e21a40d463cd27b 2024-11-12T14:49:28,991 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 9a6f7a226796c8ff7e21a40d463cd27b: Running coprocessor pre-open hook at 1731422968962Writing region info on filesystem at 1731422968962Initializing all the Stores at 1731422968963 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968963Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968964 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968964Cleaning up temporary data from old regions at 1731422968984 (+20 ms)Running coprocessor post-open hooks at 1731422968991 (+7 ms)Region opened successfully at 1731422968991 2024-11-12T14:49:28,993 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b., pid=52, masterSystemTime=1731422968613 2024-11-12T14:49:28,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=12 2024-11-12T14:49:28,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure f65e71cd750425cce9713e30a75c6002, server=3945e1ca6f47,35473,1731422964757 in 495 msec 2024-11-12T14:49:28,993 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for f943ce043844f379c1b7ecbf76b5a419 2024-11-12T14:49:28,998 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b. 2024-11-12T14:49:28,999 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b. 2024-11-12T14:49:28,999 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33. 2024-11-12T14:49:28,999 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 06aa3b2e56ae4404b602d6fa98eabe33, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-12T14:49:28,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f65e71cd750425cce9713e30a75c6002, ASSIGN in 835 msec 2024-11-12T14:49:29,000 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 06aa3b2e56ae4404b602d6fa98eabe33 2024-11-12T14:49:29,000 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:29,000 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 06aa3b2e56ae4404b602d6fa98eabe33 2024-11-12T14:49:29,000 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 06aa3b2e56ae4404b602d6fa98eabe33 2024-11-12T14:49:29,000 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f943ce043844f379c1b7ecbf76b5a419/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:49:29,001 INFO [StoreOpener-ac225c32dbf496a52e546c7e0b1a05af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region ac225c32dbf496a52e546c7e0b1a05af 2024-11-12T14:49:29,001 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened f943ce043844f379c1b7ecbf76b5a419; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72484702, jitterRate=0.0801062285900116}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:49:29,001 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f943ce043844f379c1b7ecbf76b5a419 2024-11-12T14:49:29,002 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for f943ce043844f379c1b7ecbf76b5a419: Running coprocessor pre-open hook at 1731422968971Writing region info on filesystem at 1731422968972 (+1 ms)Initializing all the Stores at 1731422968973 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968973Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968974 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968974Cleaning up temporary data from old regions at 1731422968990 (+16 ms)Running coprocessor post-open hooks at 1731422969001 (+11 ms)Region opened successfully at 1731422969001 2024-11-12T14:49:29,003 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=9a6f7a226796c8ff7e21a40d463cd27b, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,37185,1731422964568 2024-11-12T14:49:29,003 INFO [StoreOpener-06aa3b2e56ae4404b602d6fa98eabe33-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 06aa3b2e56ae4404b602d6fa98eabe33 2024-11-12T14:49:29,003 INFO [StoreOpener-ac225c32dbf496a52e546c7e0b1a05af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac225c32dbf496a52e546c7e0b1a05af columnFamilyName colfamily11 2024-11-12T14:49:29,003 DEBUG [StoreOpener-ac225c32dbf496a52e546c7e0b1a05af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:29,004 INFO [StoreOpener-ac225c32dbf496a52e546c7e0b1a05af-1 {}] regionserver.HStore(327): Store=ac225c32dbf496a52e546c7e0b1a05af/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:29,004 INFO [StoreOpener-ac225c32dbf496a52e546c7e0b1a05af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region ac225c32dbf496a52e546c7e0b1a05af 2024-11-12T14:49:29,005 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419., pid=54, masterSystemTime=1731422968761 2024-11-12T14:49:29,006 INFO [StoreOpener-06aa3b2e56ae4404b602d6fa98eabe33-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06aa3b2e56ae4404b602d6fa98eabe33 columnFamilyName colfamily11 2024-11-12T14:49:29,006 DEBUG [StoreOpener-06aa3b2e56ae4404b602d6fa98eabe33-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:29,006 INFO [StoreOpener-ac225c32dbf496a52e546c7e0b1a05af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac225c32dbf496a52e546c7e0b1a05af columnFamilyName colfamily21 2024-11-12T14:49:29,007 DEBUG [StoreOpener-ac225c32dbf496a52e546c7e0b1a05af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:29,008 INFO [StoreOpener-ac225c32dbf496a52e546c7e0b1a05af-1 {}] regionserver.HStore(327): Store=ac225c32dbf496a52e546c7e0b1a05af/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:29,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9a6f7a226796c8ff7e21a40d463cd27b, server=3945e1ca6f47,37185,1731422964568 because future has completed 2024-11-12T14:49:29,008 INFO [StoreOpener-ac225c32dbf496a52e546c7e0b1a05af-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region ac225c32dbf496a52e546c7e0b1a05af 2024-11-12T14:49:29,008 INFO [StoreOpener-06aa3b2e56ae4404b602d6fa98eabe33-1 {}] regionserver.HStore(327): Store=06aa3b2e56ae4404b602d6fa98eabe33/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:29,008 INFO [StoreOpener-06aa3b2e56ae4404b602d6fa98eabe33-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 06aa3b2e56ae4404b602d6fa98eabe33 2024-11-12T14:49:29,009 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419. 2024-11-12T14:49:29,010 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419. 2024-11-12T14:49:29,010 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=b419710b4e7b088b11da63dff9b7760e, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,45845,1731422964694 2024-11-12T14:49:29,010 INFO [StoreOpener-ac225c32dbf496a52e546c7e0b1a05af-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac225c32dbf496a52e546c7e0b1a05af columnFamilyName colfamily31 2024-11-12T14:49:29,010 DEBUG [StoreOpener-ac225c32dbf496a52e546c7e0b1a05af-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:29,010 INFO [StoreOpener-06aa3b2e56ae4404b602d6fa98eabe33-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06aa3b2e56ae4404b602d6fa98eabe33 columnFamilyName colfamily21 2024-11-12T14:49:29,010 DEBUG [StoreOpener-06aa3b2e56ae4404b602d6fa98eabe33-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:29,011 INFO [StoreOpener-ac225c32dbf496a52e546c7e0b1a05af-1 {}] regionserver.HStore(327): Store=ac225c32dbf496a52e546c7e0b1a05af/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:29,011 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for ac225c32dbf496a52e546c7e0b1a05af 2024-11-12T14:49:29,012 INFO [StoreOpener-06aa3b2e56ae4404b602d6fa98eabe33-1 {}] regionserver.HStore(327): Store=06aa3b2e56ae4404b602d6fa98eabe33/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:29,013 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac225c32dbf496a52e546c7e0b1a05af 2024-11-12T14:49:29,013 INFO [StoreOpener-06aa3b2e56ae4404b602d6fa98eabe33-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 06aa3b2e56ae4404b602d6fa98eabe33 2024-11-12T14:49:29,014 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure b419710b4e7b088b11da63dff9b7760e, server=3945e1ca6f47,45845,1731422964694 because future has completed 2024-11-12T14:49:29,014 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac225c32dbf496a52e546c7e0b1a05af 2024-11-12T14:49:29,016 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for ac225c32dbf496a52e546c7e0b1a05af 2024-11-12T14:49:29,016 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for ac225c32dbf496a52e546c7e0b1a05af 2024-11-12T14:49:29,016 INFO [StoreOpener-06aa3b2e56ae4404b602d6fa98eabe33-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06aa3b2e56ae4404b602d6fa98eabe33 columnFamilyName colfamily31 2024-11-12T14:49:29,016 DEBUG [StoreOpener-06aa3b2e56ae4404b602d6fa98eabe33-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:29,017 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:49:29,017 INFO [StoreOpener-06aa3b2e56ae4404b602d6fa98eabe33-1 {}] regionserver.HStore(327): Store=06aa3b2e56ae4404b602d6fa98eabe33/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:29,017 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 06aa3b2e56ae4404b602d6fa98eabe33 2024-11-12T14:49:29,018 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06aa3b2e56ae4404b602d6fa98eabe33 2024-11-12T14:49:29,019 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for ac225c32dbf496a52e546c7e0b1a05af 2024-11-12T14:49:29,020 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06aa3b2e56ae4404b602d6fa98eabe33 2024-11-12T14:49:29,020 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38303 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3945e1ca6f47,35473,1731422964757, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8b5e68238ca126132dbf55f0fbf38047. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T14:49:29,023 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 06aa3b2e56ae4404b602d6fa98eabe33 2024-11-12T14:49:29,023 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 06aa3b2e56ae4404b602d6fa98eabe33 2024-11-12T14:49:29,024 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:49:29,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=20 2024-11-12T14:49:29,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure b419710b4e7b088b11da63dff9b7760e, server=3945e1ca6f47,45845,1731422964694 in 637 msec 2024-11-12T14:49:29,026 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 06aa3b2e56ae4404b602d6fa98eabe33 2024-11-12T14:49:29,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=b419710b4e7b088b11da63dff9b7760e, ASSIGN in 866 msec 2024-11-12T14:49:29,028 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac225c32dbf496a52e546c7e0b1a05af/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:49:29,029 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened ac225c32dbf496a52e546c7e0b1a05af; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62034847, jitterRate=-0.0756087452173233}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:49:29,029 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ac225c32dbf496a52e546c7e0b1a05af 2024-11-12T14:49:29,030 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for ac225c32dbf496a52e546c7e0b1a05af: Running coprocessor pre-open hook at 1731422968974Writing region info on filesystem at 1731422968974Initializing all the Stores at 1731422968976 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968976Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968989 (+13 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422968989Cleaning up temporary data from old regions at 1731422969016 (+27 ms)Running coprocessor post-open hooks at 1731422969029 (+13 ms)Region opened successfully at 1731422969030 (+1 ms) 2024-11-12T14:49:29,030 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06aa3b2e56ae4404b602d6fa98eabe33/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:49:29,031 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f943ce043844f379c1b7ecbf76b5a419, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,45845,1731422964694 2024-11-12T14:49:29,032 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 06aa3b2e56ae4404b602d6fa98eabe33; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68705378, jitterRate=0.023789912462234497}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:49:29,032 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 06aa3b2e56ae4404b602d6fa98eabe33 2024-11-12T14:49:29,032 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 06aa3b2e56ae4404b602d6fa98eabe33: Running coprocessor pre-open hook at 1731422969001Writing region info on filesystem at 1731422969001Initializing all the Stores at 1731422969002 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422969002Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422969002Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422969002Cleaning up temporary data from old regions at 1731422969023 (+21 ms)Running coprocessor post-open hooks at 1731422969032 (+9 ms)Region opened successfully at 1731422969032 2024-11-12T14:49:29,034 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731422967229.ac225c32dbf496a52e546c7e0b1a05af., pid=44, masterSystemTime=1731422968641 2024-11-12T14:49:29,038 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731422967229.ac225c32dbf496a52e546c7e0b1a05af. 2024-11-12T14:49:29,038 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731422967229.ac225c32dbf496a52e546c7e0b1a05af. 2024-11-12T14:49:29,039 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33., pid=48, masterSystemTime=1731422968613 2024-11-12T14:49:29,039 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731422967229.489408be5f35a96bee1ee0f68f064599. 2024-11-12T14:49:29,039 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 489408be5f35a96bee1ee0f68f064599, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731422967229.489408be5f35a96bee1ee0f68f064599.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-12T14:49:29,039 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 489408be5f35a96bee1ee0f68f064599 2024-11-12T14:49:29,039 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731422967229.489408be5f35a96bee1ee0f68f064599.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:29,040 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 489408be5f35a96bee1ee0f68f064599 2024-11-12T14:49:29,040 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 489408be5f35a96bee1ee0f68f064599 2024-11-12T14:49:29,042 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f943ce043844f379c1b7ecbf76b5a419, server=3945e1ca6f47,45845,1731422964694 because future has completed 2024-11-12T14:49:29,042 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38303 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3945e1ca6f47,37185,1731422964568, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9a6f7a226796c8ff7e21a40d463cd27b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T14:49:29,044 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=ac225c32dbf496a52e546c7e0b1a05af, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,35473,1731422964757 2024-11-12T14:49:29,044 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33. 2024-11-12T14:49:29,045 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33. 2024-11-12T14:49:29,045 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b. 2024-11-12T14:49:29,045 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38303 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3945e1ca6f47,45845,1731422964694, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f943ce043844f379c1b7ecbf76b5a419. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T14:49:29,045 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38303 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3945e1ca6f47,45845,1731422964694, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=cf0006d9b8777107f458b7008fc42ba9. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T14:49:29,045 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 2d931ca9d93687bd79e96c94cb16690b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-12T14:49:29,046 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2d931ca9d93687bd79e96c94cb16690b 2024-11-12T14:49:29,046 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:49:29,046 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 2d931ca9d93687bd79e96c94cb16690b 2024-11-12T14:49:29,046 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 2d931ca9d93687bd79e96c94cb16690b 2024-11-12T14:49:29,048 INFO [StoreOpener-489408be5f35a96bee1ee0f68f064599-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 489408be5f35a96bee1ee0f68f064599 2024-11-12T14:49:29,050 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure ac225c32dbf496a52e546c7e0b1a05af, server=3945e1ca6f47,35473,1731422964757 because future has completed 2024-11-12T14:49:29,052 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=06aa3b2e56ae4404b602d6fa98eabe33, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,37185,1731422964568 2024-11-12T14:49:29,052 INFO [StoreOpener-489408be5f35a96bee1ee0f68f064599-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 489408be5f35a96bee1ee0f68f064599 columnFamilyName colfamily11 2024-11-12T14:49:29,052 DEBUG [StoreOpener-489408be5f35a96bee1ee0f68f064599-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:29,053 INFO [StoreOpener-489408be5f35a96bee1ee0f68f064599-1 {}] regionserver.HStore(327): Store=489408be5f35a96bee1ee0f68f064599/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:29,053 INFO [StoreOpener-489408be5f35a96bee1ee0f68f064599-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 489408be5f35a96bee1ee0f68f064599 2024-11-12T14:49:29,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06aa3b2e56ae4404b602d6fa98eabe33, server=3945e1ca6f47,37185,1731422964568 because future has completed 2024-11-12T14:49:29,058 INFO [StoreOpener-489408be5f35a96bee1ee0f68f064599-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 489408be5f35a96bee1ee0f68f064599 columnFamilyName colfamily21 2024-11-12T14:49:29,058 DEBUG [StoreOpener-489408be5f35a96bee1ee0f68f064599-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:29,059 INFO [StoreOpener-489408be5f35a96bee1ee0f68f064599-1 {}] regionserver.HStore(327): Store=489408be5f35a96bee1ee0f68f064599/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:29,059 INFO [StoreOpener-489408be5f35a96bee1ee0f68f064599-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 489408be5f35a96bee1ee0f68f064599 2024-11-12T14:49:29,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=17 2024-11-12T14:49:29,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure ac225c32dbf496a52e546c7e0b1a05af, server=3945e1ca6f47,35473,1731422964757 in 543 msec 2024-11-12T14:49:29,062 INFO [StoreOpener-489408be5f35a96bee1ee0f68f064599-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 489408be5f35a96bee1ee0f68f064599 columnFamilyName colfamily31 2024-11-12T14:49:29,062 DEBUG [StoreOpener-489408be5f35a96bee1ee0f68f064599-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:29,063 INFO [StoreOpener-489408be5f35a96bee1ee0f68f064599-1 {}] regionserver.HStore(327): Store=489408be5f35a96bee1ee0f68f064599/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:29,064 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 489408be5f35a96bee1ee0f68f064599 2024-11-12T14:49:29,065 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/489408be5f35a96bee1ee0f68f064599 2024-11-12T14:49:29,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=ac225c32dbf496a52e546c7e0b1a05af, ASSIGN in 904 msec 2024-11-12T14:49:29,065 INFO [StoreOpener-2d931ca9d93687bd79e96c94cb16690b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 2d931ca9d93687bd79e96c94cb16690b 2024-11-12T14:49:29,066 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/489408be5f35a96bee1ee0f68f064599 2024-11-12T14:49:29,068 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 489408be5f35a96bee1ee0f68f064599 2024-11-12T14:49:29,068 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 489408be5f35a96bee1ee0f68f064599 2024-11-12T14:49:29,068 INFO [StoreOpener-2d931ca9d93687bd79e96c94cb16690b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d931ca9d93687bd79e96c94cb16690b columnFamilyName colfamily11 2024-11-12T14:49:29,068 DEBUG [StoreOpener-2d931ca9d93687bd79e96c94cb16690b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:29,069 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:49:29,069 INFO [StoreOpener-2d931ca9d93687bd79e96c94cb16690b-1 {}] regionserver.HStore(327): Store=2d931ca9d93687bd79e96c94cb16690b/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:29,069 INFO [StoreOpener-2d931ca9d93687bd79e96c94cb16690b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 2d931ca9d93687bd79e96c94cb16690b 2024-11-12T14:49:29,071 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 489408be5f35a96bee1ee0f68f064599 2024-11-12T14:49:29,072 INFO [StoreOpener-2d931ca9d93687bd79e96c94cb16690b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d931ca9d93687bd79e96c94cb16690b columnFamilyName colfamily21 2024-11-12T14:49:29,072 DEBUG [StoreOpener-2d931ca9d93687bd79e96c94cb16690b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:29,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=7 2024-11-12T14:49:29,075 INFO [StoreOpener-2d931ca9d93687bd79e96c94cb16690b-1 {}] regionserver.HStore(327): Store=2d931ca9d93687bd79e96c94cb16690b/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:29,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 06aa3b2e56ae4404b602d6fa98eabe33, server=3945e1ca6f47,37185,1731422964568 in 526 msec 2024-11-12T14:49:29,075 INFO [StoreOpener-2d931ca9d93687bd79e96c94cb16690b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 2d931ca9d93687bd79e96c94cb16690b 2024-11-12T14:49:29,078 INFO [StoreOpener-2d931ca9d93687bd79e96c94cb16690b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d931ca9d93687bd79e96c94cb16690b columnFamilyName colfamily31 2024-11-12T14:49:29,078 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=06aa3b2e56ae4404b602d6fa98eabe33, ASSIGN in 917 msec 2024-11-12T14:49:29,078 DEBUG [StoreOpener-2d931ca9d93687bd79e96c94cb16690b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:49:29,078 INFO [StoreOpener-2d931ca9d93687bd79e96c94cb16690b-1 {}] regionserver.HStore(327): Store=2d931ca9d93687bd79e96c94cb16690b/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:49:29,079 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 2d931ca9d93687bd79e96c94cb16690b 2024-11-12T14:49:29,080 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2d931ca9d93687bd79e96c94cb16690b 2024-11-12T14:49:29,080 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2d931ca9d93687bd79e96c94cb16690b 2024-11-12T14:49:29,082 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/489408be5f35a96bee1ee0f68f064599/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:49:29,082 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 2d931ca9d93687bd79e96c94cb16690b 2024-11-12T14:49:29,082 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 2d931ca9d93687bd79e96c94cb16690b 2024-11-12T14:49:29,082 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 489408be5f35a96bee1ee0f68f064599; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73555092, jitterRate=0.09605628252029419}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:49:29,082 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 489408be5f35a96bee1ee0f68f064599 2024-11-12T14:49:29,083 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 489408be5f35a96bee1ee0f68f064599: Running coprocessor pre-open hook at 1731422969040Writing region info on filesystem at 1731422969040Initializing all the Stores at 1731422969041 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422969041Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422969047 (+6 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422969047Cleaning up temporary data from old regions at 1731422969068 (+21 ms)Running coprocessor post-open hooks at 1731422969083 (+15 ms)Region opened successfully at 1731422969083 2024-11-12T14:49:29,084 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:49:29,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=5 2024-11-12T14:49:29,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f943ce043844f379c1b7ecbf76b5a419, server=3945e1ca6f47,45845,1731422964694 in 471 msec 2024-11-12T14:49:29,085 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731422967229.489408be5f35a96bee1ee0f68f064599., pid=56, masterSystemTime=1731422968792 2024-11-12T14:49:29,087 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 2d931ca9d93687bd79e96c94cb16690b 2024-11-12T14:49:29,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=f943ce043844f379c1b7ecbf76b5a419, ASSIGN in 926 msec 2024-11-12T14:49:29,090 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731422967229.489408be5f35a96bee1ee0f68f064599. 2024-11-12T14:49:29,090 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731422967229.489408be5f35a96bee1ee0f68f064599. 2024-11-12T14:49:29,091 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2d931ca9d93687bd79e96c94cb16690b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:49:29,092 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 2d931ca9d93687bd79e96c94cb16690b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67214840, jitterRate=0.0015791654586791992}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:49:29,093 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2d931ca9d93687bd79e96c94cb16690b 2024-11-12T14:49:29,093 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 2d931ca9d93687bd79e96c94cb16690b: Running coprocessor pre-open hook at 1731422969046Writing region info on filesystem at 1731422969047 (+1 ms)Initializing all the Stores at 1731422969052 (+5 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422969052Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422969061 (+9 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731422969061Cleaning up temporary data from old regions at 1731422969082 (+21 ms)Running coprocessor post-open hooks at 1731422969093 (+11 ms)Region opened successfully at 1731422969093 2024-11-12T14:49:29,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=27 2024-11-12T14:49:29,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 9a6f7a226796c8ff7e21a40d463cd27b, server=3945e1ca6f47,37185,1731422964568 in 515 msec 2024-11-12T14:49:29,094 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b., pid=39, masterSystemTime=1731422968613 2024-11-12T14:49:29,096 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9a6f7a226796c8ff7e21a40d463cd27b, ASSIGN in 936 msec 2024-11-12T14:49:29,097 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b. 2024-11-12T14:49:29,097 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b. 2024-11-12T14:49:29,098 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=2d931ca9d93687bd79e96c94cb16690b, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,37185,1731422964568 2024-11-12T14:49:29,101 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d931ca9d93687bd79e96c94cb16690b, server=3945e1ca6f47,37185,1731422964568 because future has completed 2024-11-12T14:49:29,102 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=489408be5f35a96bee1ee0f68f064599, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,35473,1731422964757 2024-11-12T14:49:29,105 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 489408be5f35a96bee1ee0f68f064599, server=3945e1ca6f47,35473,1731422964757 because future has completed 2024-11-12T14:49:29,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=24 2024-11-12T14:49:29,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 2d931ca9d93687bd79e96c94cb16690b, server=3945e1ca6f47,37185,1731422964568 in 653 msec 2024-11-12T14:49:29,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2d931ca9d93687bd79e96c94cb16690b, ASSIGN in 952 msec 2024-11-12T14:49:29,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=6 2024-11-12T14:49:29,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 489408be5f35a96bee1ee0f68f064599, server=3945e1ca6f47,35473,1731422964757 in 476 msec 2024-11-12T14:49:29,120 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=489408be5f35a96bee1ee0f68f064599, ASSIGN in 960 msec 2024-11-12T14:49:29,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=23 2024-11-12T14:49:29,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 8b5e68238ca126132dbf55f0fbf38047, server=3945e1ca6f47,35473,1731422964757 in 638 msec 2024-11-12T14:49:29,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=8b5e68238ca126132dbf55f0fbf38047, ASSIGN in 969 msec 2024-11-12T14:49:29,134 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=8 2024-11-12T14:49:29,134 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure cf0006d9b8777107f458b7008fc42ba9, server=3945e1ca6f47,45845,1731422964694 in 689 msec 2024-11-12T14:49:29,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=cf0006d9b8777107f458b7008fc42ba9, ASSIGN in 976 msec 2024-11-12T14:49:29,138 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=59fa7126e2ade1f9daaaeb8bd38b09e4, regionState=OPEN, openSeqNum=2, regionLocation=3945e1ca6f47,35473,1731422964757 2024-11-12T14:49:29,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59fa7126e2ade1f9daaaeb8bd38b09e4, server=3945e1ca6f47,35473,1731422964757 because future has completed 2024-11-12T14:49:29,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=10 2024-11-12T14:49:29,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 59fa7126e2ade1f9daaaeb8bd38b09e4, server=3945e1ca6f47,35473,1731422964757 in 511 msec 2024-11-12T14:49:29,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=4 2024-11-12T14:49:29,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=59fa7126e2ade1f9daaaeb8bd38b09e4, ASSIGN in 992 msec 2024-11-12T14:49:29,155 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T14:49:29,156 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731422969155"}]},"ts":"1731422969155"} 2024-11-12T14:49:29,160 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, state=ENABLED in hbase:meta 2024-11-12T14:49:29,162 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T14:49:29,170 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true in 1.9230 sec 2024-11-12T14:49:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38303 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T14:49:29,410 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true completed 2024-11-12T14:49:29,410 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true get assigned. Timeout = 60000ms 2024-11-12T14:49:29,411 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T14:49:29,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37185 {}] regionserver.StoreScanner(1138): Switch to stream read (scanned=32802 bytes) of info 2024-11-12T14:49:29,435 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true assigned to meta. Checking AM states. 2024-11-12T14:49:29,436 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T14:49:29,437 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true assigned. 2024-11-12T14:49:29,500 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aab', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:29,504 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aac', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:29,505 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aaa', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:29,506 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aad', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:29,507 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aae', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:29,508 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aaf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:29,510 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aag', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:29,511 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='aah', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:29,580 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731422967229.489408be5f35a96bee1ee0f68f064599., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:29,582 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731422967229.489408be5f35a96bee1ee0f68f064599., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:29,587 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='bbd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731422967229.489408be5f35a96bee1ee0f68f064599., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:29,644 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:29,648 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:29,650 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:29,652 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='cce', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:29,727 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddd', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731422967229.cf0006d9b8777107f458b7008fc42ba9., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:29,729 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='dde', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731422967229.cf0006d9b8777107f458b7008fc42ba9., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:29,731 DEBUG [Time-limited test {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731422967229.cf0006d9b8777107f458b7008fc42ba9., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:29,733 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ddf', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731422967229.cf0006d9b8777107f458b7008fc42ba9., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:29,811 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eee', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:29,814 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eeg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:29,818 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eef', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:29,821 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eeh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:29,823 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eei', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:29,873 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='fff', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731422967229.59fa7126e2ade1f9daaaeb8bd38b09e4., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:29,876 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731422967229.59fa7126e2ade1f9daaaeb8bd38b09e4., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:29,879 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731422967229.59fa7126e2ade1f9daaaeb8bd38b09e4., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:29,882 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731422967229.59fa7126e2ade1f9daaaeb8bd38b09e4., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:29,884 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ffj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731422967229.59fa7126e2ade1f9daaaeb8bd38b09e4., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:29,952 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:29,955 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:29,957 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:29,963 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:29,965 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:30,028 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhh', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731422967229.f65e71cd750425cce9713e30a75c6002., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:30,032 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhi', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731422967229.f65e71cd750425cce9713e30a75c6002., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:30,037 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731422967229.f65e71cd750425cce9713e30a75c6002., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:30,043 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731422967229.f65e71cd750425cce9713e30a75c6002., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:30,046 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731422967229.f65e71cd750425cce9713e30a75c6002., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:30,048 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='hhm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731422967229.f65e71cd750425cce9713e30a75c6002., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:30,103 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iii', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731422967229.df409d6fa4df5d8fd91534d2d90eda60., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,107 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iij', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731422967229.df409d6fa4df5d8fd91534d2d90eda60., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,109 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iik', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731422967229.df409d6fa4df5d8fd91534d2d90eda60., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,111 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iil', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731422967229.df409d6fa4df5d8fd91534d2d90eda60., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,114 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iim', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731422967229.df409d6fa4df5d8fd91534d2d90eda60., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,116 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iin', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731422967229.df409d6fa4df5d8fd91534d2d90eda60., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,118 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='iio', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731422967229.df409d6fa4df5d8fd91534d2d90eda60., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,179 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:30,182 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:30,184 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:30,186 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:30,188 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:30,192 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:30,195 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:30,198 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:30,247 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkk', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731422967229.eaeef83a325d6b259b1b899f552ba8d0., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,251 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkl', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731422967229.eaeef83a325d6b259b1b899f552ba8d0., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,256 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731422967229.eaeef83a325d6b259b1b899f552ba8d0., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,260 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kkm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731422967229.eaeef83a325d6b259b1b899f552ba8d0., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,263 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='kko', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731422967229.eaeef83a325d6b259b1b899f552ba8d0., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,326 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='lll', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731422967229.51b19d6922dec6f0774f3d761593de8f., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,329 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731422967229.51b19d6922dec6f0774f3d761593de8f., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,332 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='lln', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731422967229.51b19d6922dec6f0774f3d761593de8f., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,346 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731422967229.51b19d6922dec6f0774f3d761593de8f., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,350 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='llp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731422967229.51b19d6922dec6f0774f3d761593de8f., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,407 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmm', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731422967229.ac225c32dbf496a52e546c7e0b1a05af., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:30,410 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731422967229.ac225c32dbf496a52e546c7e0b1a05af., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:30,413 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731422967229.ac225c32dbf496a52e546c7e0b1a05af., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:30,416 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731422967229.ac225c32dbf496a52e546c7e0b1a05af., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:30,420 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731422967229.ac225c32dbf496a52e546c7e0b1a05af., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:30,424 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='mmr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731422967229.ac225c32dbf496a52e546c7e0b1a05af., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:30,489 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:30,491 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nno', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:30,495 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:30,498 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:30,502 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:30,505 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nns', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:30,564 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ooo', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731422967229.bb0382e4edc08a7c43f0571deb2374a4., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:30,567 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oop', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731422967229.bb0382e4edc08a7c43f0571deb2374a4., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:30,572 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ooq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731422967229.bb0382e4edc08a7c43f0571deb2374a4., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:30,575 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oor', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731422967229.bb0382e4edc08a7c43f0571deb2374a4., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:30,577 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='oos', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731422967229.bb0382e4edc08a7c43f0571deb2374a4., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:30,637 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppp', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731422967229.b419710b4e7b088b11da63dff9b7760e., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,639 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731422967229.b419710b4e7b088b11da63dff9b7760e., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,645 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731422967229.b419710b4e7b088b11da63dff9b7760e., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,648 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='pps', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731422967229.b419710b4e7b088b11da63dff9b7760e., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,650 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ppt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731422967229.b419710b4e7b088b11da63dff9b7760e., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,701 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:30,703 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:30,710 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqs', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:30,713 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:30,754 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrr', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731422967229.a2cc208a6afe25c72a2045fb03e0ecb1., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,756 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrs', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731422967229.a2cc208a6afe25c72a2045fb03e0ecb1., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,765 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731422967229.a2cc208a6afe25c72a2045fb03e0ecb1., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,767 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rru', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731422967229.a2cc208a6afe25c72a2045fb03e0ecb1., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,770 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='rrv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731422967229.a2cc208a6afe25c72a2045fb03e0ecb1., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,806 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='sss', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731422967229.8b5e68238ca126132dbf55f0fbf38047., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:30,808 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='sst', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731422967229.8b5e68238ca126132dbf55f0fbf38047., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:30,811 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731422967229.8b5e68238ca126132dbf55f0fbf38047., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:30,813 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731422967229.8b5e68238ca126132dbf55f0fbf38047., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:30,816 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ssw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731422967229.8b5e68238ca126132dbf55f0fbf38047., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:30,859 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:30,861 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:30,863 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:30,867 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:30,870 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:30,873 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='tty', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:30,911 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuu', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731422967229.efc23c8cc5ebe3a5c96ef5d199223832., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,913 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731422967229.efc23c8cc5ebe3a5c96ef5d199223832., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,916 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731422967229.efc23c8cc5ebe3a5c96ef5d199223832., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,918 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uux', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731422967229.efc23c8cc5ebe3a5c96ef5d199223832., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,920 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='uuy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731422967229.efc23c8cc5ebe3a5c96ef5d199223832., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:30,962 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvv', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731422967229.2b04a6d83bd2998d75a1743bdf61651e., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:30,965 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvw', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731422967229.2b04a6d83bd2998d75a1743bdf61651e., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:30,972 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731422967229.2b04a6d83bd2998d75a1743bdf61651e., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:30,974 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731422967229.2b04a6d83bd2998d75a1743bdf61651e., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:30,976 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vvz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731422967229.2b04a6d83bd2998d75a1743bdf61651e., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:30,977 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vwa', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731422967229.2b04a6d83bd2998d75a1743bdf61651e., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:30,979 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vwb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731422967229.2b04a6d83bd2998d75a1743bdf61651e., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:30,980 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='vwc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731422967229.2b04a6d83bd2998d75a1743bdf61651e., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:31,029 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='www', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:31,032 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wwx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:31,037 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wwy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:31,041 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wxa', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:31,044 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wxb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:31,047 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='wwz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:31,085 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxx', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731422967229.aa9c293d6596b8e2756cafba09b3d7dd., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:31,087 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731422967229.aa9c293d6596b8e2756cafba09b3d7dd., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:31,089 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xxz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731422967229.aa9c293d6596b8e2756cafba09b3d7dd., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:31,091 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xya', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731422967229.aa9c293d6596b8e2756cafba09b3d7dd., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:31,093 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='xyb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731422967229.aa9c293d6596b8e2756cafba09b3d7dd., hostname=3945e1ca6f47,45845,1731422964694, seqNum=2] 2024-11-12T14:49:31,150 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yyy', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731422967229.95e5de757651e0aa099998abaa3b598f., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:31,152 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yyz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731422967229.95e5de757651e0aa099998abaa3b598f., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:31,154 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yza', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731422967229.95e5de757651e0aa099998abaa3b598f., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:31,155 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yzb', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731422967229.95e5de757651e0aa099998abaa3b598f., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:31,157 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='yzc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731422967229.95e5de757651e0aa099998abaa3b598f., hostname=3945e1ca6f47,35473,1731422964757, seqNum=2] 2024-11-12T14:49:31,197 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='zzz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2] 2024-11-12T14:49:31,223 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:49:31,255 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54386, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:49:31,273 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:49:31,288 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51026, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:49:31,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37185 {}] regionserver.RSRpcServices(2610): Large batch operation detected (greater than 5000) (HBASE-18023). Requested Number of Rows: 5625 Client: jenkins//172.17.0.2 first region in multi=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33. 2024-11-12T14:49:31,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35473 {}] regionserver.RSRpcServices(2610): Large batch operation detected (greater than 5000) (HBASE-18023). Requested Number of Rows: 5624 Client: jenkins//172.17.0.2 first region in multi=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731422967229.489408be5f35a96bee1ee0f68f064599. 2024-11-12T14:49:31,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45845 {}] regionserver.RSRpcServices(2610): Large batch operation detected (greater than 5000) (HBASE-18023). Requested Number of Rows: 6327 Client: jenkins//172.17.0.2 first region in multi=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419. 2024-11-12T14:49:32,240 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,, stopping at row=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true ,, for max=2147483647 with caching=100 2024-11-12T14:49:32,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37185 {}] regionserver.StoreScanner(1138): Switch to stream read (scanned=32802 bytes) of info 2024-11-12T14:49:32,292 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T14:49:32,381 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true' 2024-11-12T14:49:32,386 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-12T14:49:32,504 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: true 2024-11-12T14:49:32,504 INFO [Time-limited test {}] assignment.AssignmentTestingUtil(82): KILL REGION SERVER 3945e1ca6f47,37185,1731422964568 2024-11-12T14:49:32,505 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 3945e1ca6f47,37185,1731422964568 2024-11-12T14:49:32,505 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 3945e1ca6f47,37185,1731422964568: Simulated kill ***** 2024-11-12T14:49:32,506 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-12T14:49:32,508 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-12T14:49:32,510 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-12T14:49:32,515 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-12T14:49:32,517 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-12T14:49:32,526 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 641227264 }, "NonHeapMemoryUsage": { "committed": 158269440, "init": 7667712, "max": -1, "used": 155188536 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "3945e1ca6f47", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 1510459, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 346, "ProcessCallTime_min": 0, "ProcessCallTime_max": 515, "ProcessCallTime_mean": 6, "ProcessCallTime_25th_percentile": 128, "ProcessCallTime_median": 257, "ProcessCallTime_75th_percentile": 386, "ProcessCallTime_90th_percentile": 463, "ProcessCallTime_95th_percentile": 489, "ProcessCallTime_98th_percentile": 504, "ProcessCallTime_99th_percentile": 509, "ProcessCallTime_99.9th_percentile": 514, "ProcessCallTime_TimeRangeCount_0-1": 346, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 346, "QueueCallTime_min": 0, "QueueCallTime_max": 49, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 12, "QueueCallTime_median": 24, "QueueCallTime_75th_percentile": 36, "QueueCallTime_90th_percentile": 44, "QueueCallTime_95th_percentile": 46, "QueueCallTime_98th_percentile": 48, "QueueCallTime_99th_percentile": 48, "QueueCallTime_99.9th_percentile": 48, "QueueCallTime_TimeRangeCount_0-1": 346, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 346, "TotalCallTime_min": 0, "TotalCallTime_max": 515, "TotalCallTime_mean": 8, "TotalCallTime_25th_percentile": 128, "TotalCallTime_median": 257, "TotalCallTime_75th_percentile": 386, "TotalCallTime_90th_percentile": 463, "TotalCallTime_95th_percentile": 489, "TotalCallTime_98th_percentile": 504, "TotalCallTime_99th_percentile": 509, "TotalCallTime_99.9th_percentile": 514, "TotalCallTime_TimeRangeCount_0-1": 346, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 346, "ResponseSize_min": 0, "ResponseSize_max": 59075, "ResponseSize_mean": 13479, "ResponseSize_25th_percentile": 14768, "ResponseSize_median": 29537, "ResponseSize_75th_percentile": 44306, "ResponseSize_90th_percentile": 53167, "ResponseSize_95th_percentile": 56121, "ResponseSize_98th_percentile": 57893, "ResponseSize_99th_percentile": 58484, "ResponseSize_99.9th_percentile": 59015, "ResponseSize_SizeRangeCount_0-10": 346, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 346, "RequestSize_min": 42, "RequestSize_max": 507726, "RequestSize_mean": 4353, "RequestSize_25th_percentile": 126963, "RequestSize_median": 253884, "RequestSize_75th_percentile": 380805, "RequestSize_90th_percentile": 456957, "RequestSize_95th_percentile": 482341, "RequestSize_98th_percentile": 497572, "RequestSize_99th_percentile": 502649, "RequestSize_99.9th_percentile": 507218, "RequestSize_SizeRangeCount_0-10": 346, "sentBytes": 3453410 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "3945e1ca6f47", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:62355", "tag.serverName": "3945e1ca6f47,45845,1731422964694", "tag.clusterId": "1fc95813-043e-41c4-8f96-df8b2f6ddc4d", "tag.Context": "regionserver", "tag.Hostname": "3945e1ca6f47", "regionCount": 9, "storeCount": 27, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 6912, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731422964694, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 0, "localBytesRead": 0, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 0, "ByteBuffAllocatorPoolAllocationBytes": 1198080, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 19, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 18, "rpcScanRequestCount": 18, "rpcMultiRequestCount": 1, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 18, "ScanTime_min": 2, "ScanTime_max": 6, "ScanTime_mean": 2, "ScanTime_25th_percentile": 3, "ScanTime_median": 4, "ScanTime_75th_percentile": 5, "ScanTime_90th_percentile": 5, "ScanTime_95th_percentile": 5, "ScanTime_98th_percentile": 5, "ScanTime_99th_percentile": 5, "ScanTime_99.9th_percentile": 5, "ScanTime_TimeRangeCount_0-1": 18, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 9, "PutBatch_min": 38, "PutBatch_max": 86, "PutBatch_mean": 55, "PutBatch_25th_percentile": 50, "PutBatch_median": 62, "PutBatch_75th_percentile": 74, "PutBatch_90th_percentile": 81, "PutBatch_95th_percentile": 83, "PutBatch_98th_percentile": 85, "PutBatch_99th_percentile": 85, "PutBatch_99.9th_percentile": 85, "PutBatch_TimeRangeCount_0-1": 9, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 18, "ScanSize_min": 56240, "ScanSize_max": 56240, "ScanSize_mean": 56240, "ScanSize_25th_percentile": 56240, "ScanSize_median": 56240, "ScanSize_75th_percentile": 56240, "ScanSize_90th_percentile": 56240, "ScanSize_95th_percentile": 56240, "ScanSize_98th_percentile": 56240, "ScanSize_99th_percentile": 56240, "ScanSize_99.9th_percentile": 56240, "ScanSize_SizeRangeCount_0-10": 18, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-12T14:49:32,529 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38303 {}] master.MasterRpcServices(700): 3945e1ca6f47,37185,1731422964568 reported a fatal error: ***** ABORTING region server 3945e1ca6f47,37185,1731422964568: Simulated kill ***** 2024-11-12T14:49:32,531 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3945e1ca6f47,37185,1731422964568' ***** 2024-11-12T14:49:32,532 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-12T14:49:32,532 INFO [RS:0;3945e1ca6f47:37185 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T14:49:32,532 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T14:49:32,532 INFO [RS:0;3945e1ca6f47:37185 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-12T14:49:32,532 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T14:49:32,532 INFO [RS:0;3945e1ca6f47:37185 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-12T14:49:32,532 INFO [RS:0;3945e1ca6f47:37185 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T14:49:32,533 INFO [RS:0;3945e1ca6f47:37185 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;3945e1ca6f47:37185. 2024-11-12T14:49:32,533 DEBUG [RS:0;3945e1ca6f47:37185 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T14:49:32,537 DEBUG [RS:0;3945e1ca6f47:37185 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:49:32,538 INFO [regionserver/3945e1ca6f47:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T14:49:32,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741836_1012 (size=54214) 2024-11-12T14:49:32,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741836_1012 (size=54214) 2024-11-12T14:49:32,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741836_1012 (size=54214) 2024-11-12T14:49:32,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741833_1009 (size=298415) 2024-11-12T14:49:32,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741833_1009 (size=298415) 2024-11-12T14:49:32,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741833_1009 (size=298415) 2024-11-12T14:49:32,548 DEBUG [RS:0;3945e1ca6f47:37185 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:49:32,548 INFO [RS:0;3945e1ca6f47:37185 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T14:49:32,548 INFO [RS:0;3945e1ca6f47:37185 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37185 2024-11-12T14:49:32,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37185-0x1003469214f0001, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3945e1ca6f47,37185,1731422964568 2024-11-12T14:49:32,555 INFO [RS:0;3945e1ca6f47:37185 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T14:49:32,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T14:49:32,557 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3945e1ca6f47,37185,1731422964568] 2024-11-12T14:49:32,558 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3945e1ca6f47,37185,1731422964568 already deleted, retry=false 2024-11-12T14:49:32,559 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 3945e1ca6f47,37185,1731422964568 on 3945e1ca6f47,38303,1731422963767 2024-11-12T14:49:32,570 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true 2024-11-12T14:49:32,576 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true 2024-11-12T14:49:32,576 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=57 for 3945e1ca6f47,37185,1731422964568 (carryingMeta=true) 3945e1ca6f47,37185,1731422964568/CRASHED/regionCount=10/lock=java.util.concurrent.locks.ReentrantReadWriteLock@70929323[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-12T14:49:32,576 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-12T14:49:32,576 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true 2024-11-12T14:49:32,576 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T14:49:32,576 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T14:49:32,633 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-12T14:49:32,635 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 0 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true 2024-11-12T14:49:32,636 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T14:49:32,636 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T14:49:32,636 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T14:49:32,637 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T14:49:32,637 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T14:49:32,637 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T14:49:32,637 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T14:49:32,638 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T14:49:32,639 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T14:49:32,643 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38303 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T14:49:32,645 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T14:49:32,646 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T14:49:32,651 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-12T14:49:32,654 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true 2024-11-12T14:49:32,655 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T14:49:32,655 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 16 msec 2024-11-12T14:49:32,657 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:49:32,658 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3945e1ca6f47,37185,1731422964568, table=hbase:meta, region=1588230740 2024-11-12T14:49:32,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37185-0x1003469214f0001, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:49:32,658 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37185-0x1003469214f0001, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:49:32,658 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3945e1ca6f47,37185,1731422964568, state=OPEN 2024-11-12T14:49:32,658 INFO [RS:0;3945e1ca6f47:37185 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T14:49:32,659 INFO [RS:0;3945e1ca6f47:37185 {}] regionserver.HRegionServer(1031): Exiting; stopping=3945e1ca6f47,37185,1731422964568; zookeeper connection closed. 2024-11-12T14:49:32,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:49:32,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:49:32,659 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:49:32,660 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:49:32,660 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:49:32,660 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:49:32,665 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@23978dab {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@23978dab 2024-11-12T14:49:32,665 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731423002665 2024-11-12T14:49:32,665 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T14:49:32,665 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true 2024-11-12T14:49:32,666 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-12T14:49:32,667 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true, isMeta: true 2024-11-12T14:49:32,668 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3945e1ca6f47:37185 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:49:32,670 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:49:32,670 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568-splitting 2024-11-12T14:49:32,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T14:49:32,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:49:32,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3945e1ca6f47:37185 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:49:32,672 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3945e1ca6f47,37185,1731422964568 WAL count=1, meta=true 2024-11-12T14:49:32,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta}] 2024-11-12T14:49:32,676 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-12T14:49:32,676 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true 2024-11-12T14:49:32,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T14:49:32,676 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T14:49:32,778 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:49:32,778 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:49:32,778 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3945e1ca6f47:37185 this server is in the failed servers list 2024-11-12T14:49:32,779 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:49:32,779 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:49:32,779 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:49:32,988 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:49:32,988 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:49:32,988 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3945e1ca6f47:37185 this server is in the failed servers list 2024-11-12T14:49:32,989 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:49:32,989 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:49:32,989 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:49:33,183 INFO [regionserver/3945e1ca6f47:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-12T14:49:33,183 INFO [regionserver/3945e1ca6f47:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-12T14:49:33,298 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:49:33,298 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:49:33,299 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3945e1ca6f47:37185 this server is in the failed servers list 2024-11-12T14:49:33,299 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:49:33,299 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:49:33,299 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:49:33,808 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:49:33,808 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:49:33,808 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3945e1ca6f47:37185 this server is in the failed servers list 2024-11-12T14:49:33,809 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:49:33,809 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:49:33,809 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:49:34,207 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-12T14:49:34,207 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-12T14:49:34,210 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-12T14:49:34,210 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true Metrics about Tables on a single HBase RegionServer 2024-11-12T14:49:34,212 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T14:49:34,212 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-12T14:49:34,212 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-12T14:49:34,212 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-12T14:49:34,818 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:49:34,818 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:49:34,819 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3945e1ca6f47:37185 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:49:34,824 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3945e1ca6f47,37185,1731422964568 for '' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 60000 ms, time elapsed = 2154 ms java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-12T14:49:34,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:49:34,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T14:49:34,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:49:34,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3945e1ca6f47:37185 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:49:36,828 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:49:36,828 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:49:36,829 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3945e1ca6f47:37185 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:49:36,830 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3945e1ca6f47,37185,1731422964568 for '' in 1588230740 of hbase:meta failed, tries = 7, maxAttempts = 46, timeout = 60000 ms, time elapsed = 4163 ms java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-12T14:49:36,830 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:49:36,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T14:49:36,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:49:36,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3945e1ca6f47:37185 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:49:40,878 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:49:40,878 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:49:40,880 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3945e1ca6f47:37185 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:49:40,881 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3945e1ca6f47,37185,1731422964568 for '' in 1588230740 of hbase:meta failed, tries = 8, maxAttempts = 46, timeout = 60000 ms, time elapsed = 8214 ms java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-12T14:49:40,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:49:40,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T14:49:40,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:49:40,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3945e1ca6f47:37185 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:49:50,938 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:49:50,938 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:49:50,940 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3945e1ca6f47:37185 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:49:50,940 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3945e1ca6f47,37185,1731422964568 for '' in 1588230740 of hbase:meta failed, tries = 9, maxAttempts = 46, timeout = 60000 ms, time elapsed = 18274 ms java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-12T14:49:50,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:49:50,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T14:49:50,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:49:50,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3945e1ca6f47:37185 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:49:52,850 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T14:49:59,833 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=3945e1ca6f47:38303 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-12T14:50:00,948 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:50:00,948 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:50:00,949 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3945e1ca6f47:37185 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:50:00,950 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3945e1ca6f47,37185,1731422964568 for '' in 1588230740 of hbase:meta failed, tries = 10, maxAttempts = 46, timeout = 60000 ms, time elapsed = 28283 ms java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-12T14:50:00,950 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:50:00,950 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T14:50:00,950 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:50:00,950 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3945e1ca6f47:37185 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:50:04,171 WARN [RS:2;3945e1ca6f47:35473 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of 3945e1ca6f47,37185,1731422964568: status=status unset, state=RUNNING, startTime=1731422972575, completionTime=-1 appears to have been leaked 2024-11-12T14:50:07,117 INFO [master/3945e1ca6f47:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-12T14:50:07,117 INFO [master/3945e1ca6f47:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-12T14:50:11,048 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:50:11,048 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:50:11,050 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3945e1ca6f47:37185 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:50:11,051 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3945e1ca6f47,37185,1731422964568 for '' in 1588230740 of hbase:meta failed, tries = 11, maxAttempts = 46, timeout = 60000 ms, time elapsed = 38384 ms java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-12T14:50:11,051 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:50:11,052 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T14:50:11,052 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:50:11,052 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3945e1ca6f47:37185 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:50:21,088 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:50:21,088 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:50:21,089 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3945e1ca6f47:37185 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:50:21,090 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3945e1ca6f47,37185,1731422964568 for '' in 1588230740 of hbase:meta failed, tries = 12, maxAttempts = 46, timeout = 60000 ms, time elapsed = 48424 ms java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-12T14:50:21,091 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:50:21,091 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T14:50:21,091 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:50:21,092 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3945e1ca6f47:37185 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:50:22,850 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T14:50:27,457 INFO [regionserver/3945e1ca6f47:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: ExecutorStatusChore was stopped 2024-11-12T14:50:28,944 DEBUG [master/3945e1ca6f47:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 489408be5f35a96bee1ee0f68f064599 changed from -1.0 to 0.0, refreshing cache 2024-11-12T14:50:28,945 DEBUG [master/3945e1ca6f47:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 59fa7126e2ade1f9daaaeb8bd38b09e4 changed from -1.0 to 0.0, refreshing cache 2024-11-12T14:50:28,946 DEBUG [master/3945e1ca6f47:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f65e71cd750425cce9713e30a75c6002 changed from -1.0 to 0.0, refreshing cache 2024-11-12T14:50:28,946 DEBUG [master/3945e1ca6f47:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ac225c32dbf496a52e546c7e0b1a05af changed from -1.0 to 0.0, refreshing cache 2024-11-12T14:50:28,947 DEBUG [master/3945e1ca6f47:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region bb0382e4edc08a7c43f0571deb2374a4 changed from -1.0 to 0.0, refreshing cache 2024-11-12T14:50:28,947 DEBUG [master/3945e1ca6f47:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8b5e68238ca126132dbf55f0fbf38047 changed from -1.0 to 0.0, refreshing cache 2024-11-12T14:50:28,948 DEBUG [master/3945e1ca6f47:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2b04a6d83bd2998d75a1743bdf61651e changed from -1.0 to 0.0, refreshing cache 2024-11-12T14:50:28,948 DEBUG [master/3945e1ca6f47:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 95e5de757651e0aa099998abaa3b598f changed from -1.0 to 0.0, refreshing cache 2024-11-12T14:50:28,948 DEBUG [master/3945e1ca6f47:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f943ce043844f379c1b7ecbf76b5a419 changed from -1.0 to 0.0, refreshing cache 2024-11-12T14:50:28,949 DEBUG [master/3945e1ca6f47:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region cf0006d9b8777107f458b7008fc42ba9 changed from -1.0 to 0.0, refreshing cache 2024-11-12T14:50:28,949 DEBUG [master/3945e1ca6f47:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region df409d6fa4df5d8fd91534d2d90eda60 changed from -1.0 to 0.0, refreshing cache 2024-11-12T14:50:28,949 DEBUG [master/3945e1ca6f47:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region eaeef83a325d6b259b1b899f552ba8d0 changed from -1.0 to 0.0, refreshing cache 2024-11-12T14:50:28,950 DEBUG [master/3945e1ca6f47:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 51b19d6922dec6f0774f3d761593de8f changed from -1.0 to 0.0, refreshing cache 2024-11-12T14:50:28,950 DEBUG [master/3945e1ca6f47:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b419710b4e7b088b11da63dff9b7760e changed from -1.0 to 0.0, refreshing cache 2024-11-12T14:50:28,950 DEBUG [master/3945e1ca6f47:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a2cc208a6afe25c72a2045fb03e0ecb1 changed from -1.0 to 0.0, refreshing cache 2024-11-12T14:50:28,950 DEBUG [master/3945e1ca6f47:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region efc23c8cc5ebe3a5c96ef5d199223832 changed from -1.0 to 0.0, refreshing cache 2024-11-12T14:50:28,951 DEBUG [master/3945e1ca6f47:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region aa9c293d6596b8e2756cafba09b3d7dd changed from -1.0 to 0.0, refreshing cache 2024-11-12T14:50:31,839 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=3945e1ca6f47:38303 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-12T14:50:32,669 WARN [Time-limited test {}] procedure.MasterProcedureTestingUtility$3(130): Failed to load meta org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=13, exceptions: 2024-11-12T14:49:32.671Z, java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:49:32.779Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:49:32.989Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:49:33.299Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:49:33.809Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:49:34.825Z, java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:49:36.831Z, java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:49:40.882Z, java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:49:50.942Z, java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:50:00.950Z, java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:50:11.052Z, java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:50:21.091Z, java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:493) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:461) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:422) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.fullScanRegions(MetaTableAccessor.java:108) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionStateStore.visitMeta(RegionStateStore.java:108) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.loadMeta(AssignmentManager.java:1907) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.joinCluster(AssignmentManager.java:1769) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:121) ~[test-classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:143) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:477) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:486) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPBase.testRecoveryAndDoubleExecution(TestSCPBase.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta.testRecoveryAndDoubleExecutionOnRsWithMeta(TestSCPWithMeta.java:36) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:102) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-12T14:50:32,672 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 1 exec state=pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true 2024-11-12T14:50:32,672 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T14:50:32,672 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T14:50:32,672 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T14:50:32,673 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T14:50:32,673 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T14:50:32,673 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T14:50:32,673 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T14:50:32,674 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T14:50:32,675 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-12T14:50:32,679 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T14:50:32,679 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T14:50:32,680 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-12T14:50:32,681 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=false; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true 2024-11-12T14:50:32,681 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=false; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T14:50:32,681 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-12T14:50:32,682 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:50:32,682 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3945e1ca6f47,37185,1731422964568, table=hbase:meta, region=1588230740 2024-11-12T14:50:32,683 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3945e1ca6f47,37185,1731422964568, state=OPEN 2024-11-12T14:50:32,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:50:32,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:50:32,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:50:32,684 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:50:32,684 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:50:32,684 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:50:32,685 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731423062685 2024-11-12T14:50:32,685 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T14:50:32,685 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true, isMeta: true 2024-11-12T14:50:32,685 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:50:32,685 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:50:32,686 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3945e1ca6f47:37185 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:50:32,686 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:50:32,686 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T14:50:32,686 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:50:32,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3945e1ca6f47:37185 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:50:32,687 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3945e1ca6f47,37185,1731422964568 WAL count=1, meta=true 2024-11-12T14:50:32,688 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta}] 2024-11-12T14:50:32,688 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-12T14:50:32,693 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=3945e1ca6f47,35473,1731422964757 2024-11-12T14:50:32,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757}] 2024-11-12T14:50:32,798 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:50:32,798 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:50:32,799 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3945e1ca6f47:37185 this server is in the failed servers list 2024-11-12T14:50:32,800 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:50:32,800 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:50:32,800 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:50:32,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-11-12T14:50:32,886 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568-splitting/3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, size=52.9 K (54214bytes) 2024-11-12T14:50:32,887 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-12T14:50:32,887 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-12T14:50:32,887 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568-splitting/3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta 2024-11-12T14:50:32,892 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568-splitting/3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta after 3ms 2024-11-12T14:50:32,899 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568-splitting/3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta took 14ms 2024-11-12T14:50:32,907 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 2024-11-12T14:50:32,914 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta.temp 2024-11-12T14:50:32,916 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568-splitting/3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta so closing down 2024-11-12T14:50:32,916 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T14:50:32,922 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta.temp 2024-11-12T14:50:32,925 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T14:50:32,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741864_1040 (size=53973) 2024-11-12T14:50:32,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741864_1040 (size=53973) 2024-11-12T14:50:32,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741864_1040 (size=53973) 2024-11-12T14:50:33,007 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:50:33,007 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:50:33,008 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3945e1ca6f47:37185 this server is in the failed servers list 2024-11-12T14:50:33,008 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:50:33,008 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:50:33,008 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:50:33,318 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:50:33,318 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:50:33,318 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3945e1ca6f47:37185 this server is in the failed servers list 2024-11-12T14:50:33,319 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:50:33,319 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:50:33,319 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:50:33,343 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) 2024-11-12T14:50:33,346 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta.temp to hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-12T14:50:33,349 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 446 ms; skipped=1; WAL=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568-splitting/3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, size=52.9 K, length=54214, corrupted=false, cancelled=false 2024-11-12T14:50:33,349 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568-splitting/3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, journal: Splitting hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568-splitting/3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, size=52.9 K (54214bytes) at 1731423032886Finishing writing output for hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568-splitting/3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta so closing down at 1731423032916 (+30 ms)Creating recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta.temp at 1731423032922 (+6 ms)3 split writer threads finished at 1731423032925 (+3 ms)Closed recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) at 1731423033343 (+418 ms)Rename recovered edits hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta.temp to hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1731423033346 (+3 ms)Processed 58 edits across 1 Regions in 446 ms; skipped=1; WAL=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568-splitting/3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, size=52.9 K, length=54214, corrupted=false, cancelled=false at 1731423033349 (+3 ms) 2024-11-12T14:50:33,350 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568-splitting/3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta 2024-11-12T14:50:33,351 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-12T14:50:33,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38303 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-12T14:50:33,359 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568-splitting/3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta to hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/oldWALs 2024-11-12T14:50:33,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-12T14:50:33,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757 in 665 msec 2024-11-12T14:50:33,363 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=3945e1ca6f47,35473,1731422964757 2024-11-12T14:50:33,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-12T14:50:33,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757 in 676 msec 2024-11-12T14:50:33,367 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568-splitting dir is empty, no logs to split. 2024-11-12T14:50:33,368 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3945e1ca6f47,37185,1731422964568 WAL count=0, meta=true 2024-11-12T14:50:33,368 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 3945e1ca6f47,37185,1731422964568 WAL splitting is done? wals=0, meta=true 2024-11-12T14:50:33,368 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-12T14:50:33,368 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true 2024-11-12T14:50:33,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T14:50:33,368 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T14:50:33,827 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:50:33,828 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:50:33,828 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3945e1ca6f47:37185 this server is in the failed servers list 2024-11-12T14:50:33,829 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:50:33,829 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:50:33,830 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:50:34,838 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:50:34,838 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:50:34,839 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3945e1ca6f47:37185 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:50:34,839 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3945e1ca6f47,37185,1731422964568 for '' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 60000 ms, time elapsed = 2153 ms java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-12T14:50:34,840 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:50:34,840 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T14:50:34,840 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:50:34,840 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3945e1ca6f47:37185 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:50:36,857 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:50:36,858 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:50:36,859 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3945e1ca6f47:37185 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:50:36,860 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3945e1ca6f47,37185,1731422964568 for '' in 1588230740 of hbase:meta failed, tries = 7, maxAttempts = 46, timeout = 60000 ms, time elapsed = 4175 ms java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-12T14:50:36,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:50:36,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T14:50:36,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:50:36,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3945e1ca6f47:37185 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:50:40,878 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:50:40,878 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:50:40,880 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3945e1ca6f47:37185 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:50:40,880 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3945e1ca6f47,37185,1731422964568 for '' in 1588230740 of hbase:meta failed, tries = 8, maxAttempts = 46, timeout = 60000 ms, time elapsed = 8195 ms java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-12T14:50:40,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:50:40,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T14:50:40,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:50:40,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3945e1ca6f47:37185 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:50:50,958 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:50:50,958 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:50:50,960 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3945e1ca6f47:37185 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:50:50,961 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3945e1ca6f47,37185,1731422964568 for '' in 1588230740 of hbase:meta failed, tries = 9, maxAttempts = 46, timeout = 60000 ms, time elapsed = 18275 ms java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-12T14:50:50,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:50:50,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T14:50:50,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:50:50,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3945e1ca6f47:37185 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:50:52,851 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T14:51:01,047 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:51:01,047 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:51:01,048 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3945e1ca6f47:37185 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:51:01,048 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3945e1ca6f47,37185,1731422964568 for '' in 1588230740 of hbase:meta failed, tries = 10, maxAttempts = 46, timeout = 60000 ms, time elapsed = 28363 ms java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-12T14:51:01,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:51:01,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T14:51:01,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:51:01,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3945e1ca6f47:37185 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:51:11,058 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:51:11,058 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:51:11,059 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3945e1ca6f47:37185 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:51:11,060 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3945e1ca6f47,37185,1731422964568 for '' in 1588230740 of hbase:meta failed, tries = 11, maxAttempts = 46, timeout = 60000 ms, time elapsed = 38374 ms java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-12T14:51:11,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:51:11,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T14:51:11,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:51:11,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3945e1ca6f47:37185 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:51:21,138 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:51:21,138 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:51:21,139 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3945e1ca6f47:37185 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:51:21,140 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3945e1ca6f47,37185,1731422964568 for '' in 1588230740 of hbase:meta failed, tries = 12, maxAttempts = 46, timeout = 60000 ms, time elapsed = 48454 ms java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-12T14:51:21,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:51:21,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T14:51:21,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:51:21,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3945e1ca6f47:37185 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:51:22,851 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T14:51:29,157 INFO [regionserver/3945e1ca6f47:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactedHFilesCleaner was stopped 2024-11-12T14:51:32,689 WARN [Time-limited test {}] procedure.MasterProcedureTestingUtility$3(130): Failed to load meta org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=13, exceptions: 2024-11-12T14:50:32.686Z, java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:50:32.800Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:50:33.008Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:50:33.319Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:50:33.830Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:50:34.840Z, java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:50:36.862Z, java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:50:40.881Z, java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:50:50.962Z, java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:51:01.049Z, java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:51:11.060Z, java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:51:21.140Z, java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:493) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:461) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:422) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.fullScanRegions(MetaTableAccessor.java:108) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionStateStore.visitMeta(RegionStateStore.java:108) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.loadMeta(AssignmentManager.java:1907) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.joinCluster(AssignmentManager.java:1769) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:121) ~[test-classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:143) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:477) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:486) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPBase.testRecoveryAndDoubleExecution(TestSCPBase.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta.testRecoveryAndDoubleExecutionOnRsWithMeta(TestSCPWithMeta.java:36) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:102) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-12T14:51:32,691 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 2 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true 2024-11-12T14:51:32,691 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T14:51:32,691 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T14:51:32,691 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T14:51:32,691 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T14:51:32,692 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T14:51:32,692 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T14:51:32,692 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T14:51:32,693 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T14:51:32,694 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-12T14:51:32,697 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T14:51:32,697 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T14:51:32,697 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T14:51:32,697 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T14:51:32,699 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-12T14:51:32,699 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=false; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true 2024-11-12T14:51:32,699 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757 2024-11-12T14:51:32,699 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757 2024-11-12T14:51:32,699 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, hasLock=false; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T14:51:32,699 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:51:32,699 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:51:32,700 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-12T14:51:32,700 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:51:32,700 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3945e1ca6f47,37185,1731422964568, table=hbase:meta, region=1588230740 2024-11-12T14:51:32,701 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3945e1ca6f47,37185,1731422964568, state=OPEN 2024-11-12T14:51:32,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:51:32,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:51:32,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:51:32,702 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:51:32,702 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:51:32,702 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:51:32,703 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731423122703 2024-11-12T14:51:32,703 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T14:51:32,703 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:51:32,703 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:51:32,704 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3945e1ca6f47:37185 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:51:32,705 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:51:32,705 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T14:51:32,705 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:51:32,705 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3945e1ca6f47:37185 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:51:32,705 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568-splitting dir is empty, no logs to split. 2024-11-12T14:51:32,705 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3945e1ca6f47,37185,1731422964568 WAL count=0, meta=true 2024-11-12T14:51:32,705 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 3945e1ca6f47,37185,1731422964568 WAL splitting is done? wals=0, meta=true 2024-11-12T14:51:32,705 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-12T14:51:32,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T14:51:32,707 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-12T14:51:32,707 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_GET_REGIONS, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true 2024-11-12T14:51:32,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T14:51:32,707 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_GET_REGIONS, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T14:51:32,808 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:51:32,808 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:51:32,808 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3945e1ca6f47:37185 this server is in the failed servers list 2024-11-12T14:51:32,808 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:51:32,809 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:51:32,809 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:51:33,018 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:51:33,018 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:51:33,019 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3945e1ca6f47:37185 this server is in the failed servers list 2024-11-12T14:51:33,020 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:51:33,020 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:51:33,020 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:51:33,328 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:51:33,328 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:51:33,329 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3945e1ca6f47:37185 this server is in the failed servers list 2024-11-12T14:51:33,330 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:51:33,330 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:51:33,330 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:51:33,838 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:51:33,838 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:51:33,838 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3945e1ca6f47:37185 this server is in the failed servers list 2024-11-12T14:51:33,839 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:51:33,839 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:51:33,839 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:51:34,847 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:51:34,848 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:51:34,849 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3945e1ca6f47:37185 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:51:34,849 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3945e1ca6f47,37185,1731422964568 for '' in 1588230740 of hbase:meta failed, tries = 6, maxAttempts = 46, timeout = 60000 ms, time elapsed = 2146 ms java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-12T14:51:34,850 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:51:34,850 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T14:51:34,850 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:51:34,850 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3945e1ca6f47:37185 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:51:36,867 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:51:36,868 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:51:36,868 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3945e1ca6f47:37185 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:51:36,869 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3945e1ca6f47,37185,1731422964568 for '' in 1588230740 of hbase:meta failed, tries = 7, maxAttempts = 46, timeout = 60000 ms, time elapsed = 4165 ms java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-12T14:51:36,869 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:51:36,869 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T14:51:36,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:51:36,870 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3945e1ca6f47:37185 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:51:37,842 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=3945e1ca6f47:38303 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 10 more 2024-11-12T14:51:40,888 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:51:40,888 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:51:40,890 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3945e1ca6f47:37185 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:51:40,890 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3945e1ca6f47,37185,1731422964568 for '' in 1588230740 of hbase:meta failed, tries = 8, maxAttempts = 46, timeout = 60000 ms, time elapsed = 8187 ms java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-12T14:51:40,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:51:40,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T14:51:40,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:51:40,892 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3945e1ca6f47:37185 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:51:50,917 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:51:50,918 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:51:50,919 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3945e1ca6f47:37185 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:51:50,920 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3945e1ca6f47,37185,1731422964568 for '' in 1588230740 of hbase:meta failed, tries = 9, maxAttempts = 46, timeout = 60000 ms, time elapsed = 18216 ms java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-12T14:51:50,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:51:50,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T14:51:50,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:51:50,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3945e1ca6f47:37185 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:51:52,852 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T14:52:00,988 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:52:00,988 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:52:00,990 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3945e1ca6f47:37185 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:52:00,991 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3945e1ca6f47,37185,1731422964568 for '' in 1588230740 of hbase:meta failed, tries = 10, maxAttempts = 46, timeout = 60000 ms, time elapsed = 28288 ms java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-12T14:52:00,993 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:52:00,993 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T14:52:00,993 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:52:00,993 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3945e1ca6f47:37185 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:52:07,249 WARN [RS:2;3945e1ca6f47:35473 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of 3945e1ca6f47,37185,1731422964568: status=Processing ServerCrashProcedure of 3945e1ca6f47,37185,1731422964568 current State SERVER_CRASH_SPLIT_META_LOGS, state=RUNNING, startTime=1731422972654, completionTime=-1 appears to have been leaked 2024-11-12T14:52:07,249 WARN [RS:2;3945e1ca6f47:35473 {}] monitoring.TaskMonitor(166): Status Processing ServerCrashProcedure of 3945e1ca6f47,37185,1731422964568: status=Processing ServerCrashProcedure of 3945e1ca6f47,37185,1731422964568 current State SERVER_CRASH_DELETE_SPLIT_META_WALS_DIR, state=RUNNING, startTime=1731423032680, completionTime=-1 appears to have been leaked 2024-11-12T14:52:11,048 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:52:11,048 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:52:11,050 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3945e1ca6f47:37185 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:52:11,051 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3945e1ca6f47,37185,1731422964568 for '' in 1588230740 of hbase:meta failed, tries = 11, maxAttempts = 46, timeout = 60000 ms, time elapsed = 38347 ms java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-12T14:52:11,052 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:52:11,052 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T14:52:11,052 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:52:11,052 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3945e1ca6f47:37185 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:52:21,067 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:52:21,067 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:52:21,068 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3945e1ca6f47:37185 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:52:21,069 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRpcRetryingCaller(177): Call to 3945e1ca6f47,37185,1731422964568 for '' in 1588230740 of hbase:meta failed, tries = 12, maxAttempts = 46, timeout = 60000 ms, time elapsed = 48365 ms java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] ... 7 more 2024-11-12T14:52:21,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:52:21,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T14:52:21,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:52:21,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3945e1ca6f47:37185 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:52:22,852 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T14:52:32,708 WARN [Time-limited test {}] procedure.MasterProcedureTestingUtility$3(130): Failed to load meta org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=13, exceptions: 2024-11-12T14:51:32.705Z, java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:51:32.809Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:51:33.020Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:51:33.330Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:51:33.839Z, org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:51:34.850Z, java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:51:36.870Z, java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:51:40.891Z, java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:51:50.921Z, java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:52:00.993Z, java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:52:11.052Z, java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:52:21.070Z, java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncTableResultScanner.next(AsyncTableResultScanner.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:493) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:461) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.scanMeta(MetaTableAccessor.java:422) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.MetaTableAccessor.fullScanRegions(MetaTableAccessor.java:108) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionStateStore.visitMeta(RegionStateStore.java:108) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.loadMeta(AssignmentManager.java:1907) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.joinCluster(AssignmentManager.java:1769) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility$3.call(MasterProcedureTestingUtility.java:121) ~[test-classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:143) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT-tests.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:477) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(MasterProcedureTestingUtility.java:486) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPBase.testRecoveryAndDoubleExecution(TestSCPBase.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.TestSCPWithMeta.testRecoveryAndDoubleExecutionOnRsWithMeta(TestSCPWithMeta.java:36) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.completeExceptionally(AsyncRpcRetryingCaller.java:109) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:102) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:220) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.BufferCallBeforeInitHandler.userEventTriggered(BufferCallBeforeInitHandler.java:106) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:398) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireUserEventTriggered(AbstractChannelHandlerContext.java:368) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.userEventTriggered(DefaultChannelPipeline.java:1375) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:396) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeUserEventTriggered(AbstractChannelHandlerContext.java:376) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireUserEventTriggered(DefaultChannelPipeline.java:862) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcConnection.failInit(NettyRpcConnection.java:210) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.fail(NettyRpcConnection.java:414) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:421) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcConnection$2.operationComplete(NettyRpcConnection.java:389) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:629) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:118) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:679) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:698) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more Caused by: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-12T14:52:32,710 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 3 exec state=pid=57, state=WAITING:SERVER_CRASH_GET_REGIONS, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true 2024-11-12T14:52:32,710 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T14:52:32,710 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T14:52:32,710 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T14:52:32,710 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T14:52:32,710 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T14:52:32,710 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T14:52:32,710 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T14:52:32,710 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T14:52:32,711 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T14:52:32,713 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T14:52:32,714 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T14:52:32,714 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T14:52:32,714 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T14:52:32,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-12T14:52:32,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=false; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true 2024-11-12T14:52:32,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:32,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:32,715 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN_META, hasLock=false; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T14:52:32,715 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:32,716 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:32,716 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-12T14:52:32,716 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:32,717 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3945e1ca6f47,37185,1731422964568, table=hbase:meta, region=1588230740 2024-11-12T14:52:32,717 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3945e1ca6f47,37185,1731422964568, state=OPEN 2024-11-12T14:52:32,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:52:32,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:52:32,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:52:32,718 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:52:32,718 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:52:32,718 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:52:32,719 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731423182719 2024-11-12T14:52:32,719 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T14:52:32,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T14:52:32,719 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-12T14:52:32,720 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:52:32,720 DEBUG [Time-limited test {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:52:32,721 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3945e1ca6f47:37185 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:52:32,721 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:52:32,721 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-12T14:52:32,721 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:52:32,721 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 3945e1ca6f47:37185 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:52:32,722 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T14:52:32,722 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T14:52:32,828 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:52:32,828 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1] 2024-11-12T14:52:32,828 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3945e1ca6f47:37185 this server is in the failed servers list 2024-11-12T14:52:32,829 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:52:32,829 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:52:32,829 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:52:32,873 DEBUG [3945e1ca6f47:38303 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-12T14:52:32,873 DEBUG [3945e1ca6f47:38303 {}] balancer.BalancerClusterState(204): Hosts are {3945e1ca6f47=0} racks are {/default-rack=0} 2024-11-12T14:52:32,873 DEBUG [3945e1ca6f47:38303 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T14:52:32,873 DEBUG [3945e1ca6f47:38303 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T14:52:32,874 DEBUG [3945e1ca6f47:38303 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T14:52:32,874 DEBUG [3945e1ca6f47:38303 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T14:52:32,874 INFO [3945e1ca6f47:38303 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T14:52:32,874 INFO [3945e1ca6f47:38303 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T14:52:32,874 DEBUG [3945e1ca6f47:38303 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T14:52:32,874 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=3945e1ca6f47,45845,1731422964694 2024-11-12T14:52:32,875 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3945e1ca6f47,45845,1731422964694, state=OPENING 2024-11-12T14:52:32,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:52:32,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:52:32,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:52:32,876 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T14:52:32,876 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:52:32,876 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:52:32,876 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:52:32,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=3945e1ca6f47,45845,1731422964694}] 2024-11-12T14:52:33,030 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49883, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T14:52:33,037 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-12T14:52:33,037 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:52:33,037 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-12T14:52:33,038 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:52:33,038 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,45845,1731422964694, seqNum=-1] 2024-11-12T14:52:33,038 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:52:33,039 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52255, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:52:33,040 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=3945e1ca6f47%2C45845%2C1731422964694.meta, suffix=.meta, logDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,45845,1731422964694, archiveDir=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/oldWALs, maxLogs=32 2024-11-12T14:52:33,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45845 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Scan size: 121 connection: 172.17.0.2:52255 deadline: 1731423212721, exception=org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 3945e1ca6f47,45845,1731422964694 2024-11-12T14:52:33,045 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,45845,1731422964694, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,45845,1731422964694, seqNum=-1, error=org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 3945e1ca6f47,45845,1731422964694 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:52:33,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,45845,1731422964694, seqNum=-1 is org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 3945e1ca6f47,45845,1731422964694 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:52:33,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,45845,1731422964694, seqNum=-1 from cache 2024-11-12T14:52:33,055 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,45845,1731422964694/3945e1ca6f47%2C45845%2C1731422964694.meta.1731423153040.meta, exclude list is [], retry=0 2024-11-12T14:52:33,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44401,DS-48b2b78b-5966-4c15-86cf-f894eccf2d5b,DISK] 2024-11-12T14:52:33,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38947,DS-2fd665aa-69eb-44fd-bca0-32d25d7b0e8b,DISK] 2024-11-12T14:52:33,059 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41429,DS-7b068bce-ec51-4d06-8fdd-88c861866bd0,DISK] 2024-11-12T14:52:33,105 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,45845,1731422964694/3945e1ca6f47%2C45845%2C1731422964694.meta.1731423153040.meta 2024-11-12T14:52:33,106 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34963:34963),(127.0.0.1/127.0.0.1:38447:38447),(127.0.0.1/127.0.0.1:36551:36551)] 2024-11-12T14:52:33,106 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:52:33,106 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-12T14:52:33,106 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-12T14:52:33,107 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-12T14:52:33,107 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-12T14:52:33,107 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:52:33,107 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-12T14:52:33,107 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-12T14:52:33,112 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T14:52:33,114 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T14:52:33,114 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:33,115 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:52:33,115 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T14:52:33,116 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T14:52:33,116 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:33,117 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:52:33,117 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T14:52:33,118 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T14:52:33,118 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:33,119 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:52:33,119 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T14:52:33,120 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T14:52:33,120 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:33,120 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:52:33,120 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T14:52:33,121 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740 2024-11-12T14:52:33,124 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740 2024-11-12T14:52:33,125 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-12T14:52:33,142 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-12T14:52:33,148 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=49.58 KB heapSize=69.98 KB 2024-11-12T14:52:33,231 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/.tmp/info/eb1a19a0918b4c02b8510683ffebabe1 is 268, key is testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731422967229.489408be5f35a96bee1ee0f68f064599./info:regioninfo/1731422969102/Put/seqid=0 2024-11-12T14:52:33,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741866_1042 (size=58053) 2024-11-12T14:52:33,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741866_1042 (size=58053) 2024-11-12T14:52:33,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741866_1042 (size=58053) 2024-11-12T14:52:33,261 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.31 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/.tmp/info/eb1a19a0918b4c02b8510683ffebabe1 2024-11-12T14:52:33,353 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/.tmp/ns/4a25d56ad5d4439b997a472230af343f is 43, key is default/ns:d/1731422966932/Put/seqid=0 2024-11-12T14:52:33,358 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:52:33,358 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,45845,1731422964694, seqNum=-1] 2024-11-12T14:52:33,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45845 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Scan size: 121 connection: 172.17.0.2:52255 deadline: 1731423212720, exception=org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 3945e1ca6f47,45845,1731422964694 2024-11-12T14:52:33,360 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,45845,1731422964694, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,45845,1731422964694, seqNum=-1, error=org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 3945e1ca6f47,45845,1731422964694 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:52:33,360 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,45845,1731422964694, seqNum=-1 is org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 3945e1ca6f47,45845,1731422964694 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkQuotaAndGetRegionScannerContext(RSRpcServices.java:4010) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3577) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:52:33,360 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,45845,1731422964694, seqNum=-1 from cache 2024-11-12T14:52:33,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741867_1043 (size=5153) 2024-11-12T14:52:33,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741867_1043 (size=5153) 2024-11-12T14:52:33,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741867_1043 (size=5153) 2024-11-12T14:52:33,363 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/.tmp/ns/4a25d56ad5d4439b997a472230af343f 2024-11-12T14:52:33,392 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/.tmp/table/116d882478834b8293792215d37109b7 is 105, key is testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/table:state/1731422969155/Put/seqid=0 2024-11-12T14:52:33,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741868_1044 (size=5620) 2024-11-12T14:52:33,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741868_1044 (size=5620) 2024-11-12T14:52:33,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741868_1044 (size=5620) 2024-11-12T14:52:33,408 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=202 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/.tmp/table/116d882478834b8293792215d37109b7 2024-11-12T14:52:33,419 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/.tmp/info/eb1a19a0918b4c02b8510683ffebabe1 as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/info/eb1a19a0918b4c02b8510683ffebabe1 2024-11-12T14:52:33,435 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/info/eb1a19a0918b4c02b8510683ffebabe1, entries=260, sequenceid=60, filesize=56.7 K 2024-11-12T14:52:33,439 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/.tmp/ns/4a25d56ad5d4439b997a472230af343f as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/ns/4a25d56ad5d4439b997a472230af343f 2024-11-12T14:52:33,455 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/ns/4a25d56ad5d4439b997a472230af343f, entries=2, sequenceid=60, filesize=5.0 K 2024-11-12T14:52:33,457 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/.tmp/table/116d882478834b8293792215d37109b7 as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/table/116d882478834b8293792215d37109b7 2024-11-12T14:52:33,468 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/table/116d882478834b8293792215d37109b7, entries=2, sequenceid=60, filesize=5.5 K 2024-11-12T14:52:33,474 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~49.58 KB/50772, heapSize ~69.69 KB/71360, currentSize=0 B/0 for 1588230740 in 327ms, sequenceid=60, compaction requested=false; wal=null 2024-11-12T14:52:33,476 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-12T14:52:33,478 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T14:52:33,478 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T14:52:33,480 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T14:52:33,482 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T14:52:33,489 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-12T14:52:33,491 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66683932, jitterRate=-0.006331980228424072}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T14:52:33,491 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-12T14:52:33,492 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731423153107Writing region info on filesystem at 1731423153107Initializing all the Stores at 1731423153109 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731423153109Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731423153112 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423153112Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731423153112Obtaining lock to block concurrent updates at 1731423153148 (+36 ms)Preparing flush snapshotting stores in 1588230740 at 1731423153148Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=50772, getHeapSize=71600, getOffHeapSize=0, getCellsCount=264 at 1731423153152 (+4 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731423153152Flushing 1588230740/info: creating writer at 1731423153157 (+5 ms)Flushing 1588230740/info: appending metadata at 1731423153221 (+64 ms)Flushing 1588230740/info: closing flushed file at 1731423153224 (+3 ms)Flushing 1588230740/ns: creating writer at 1731423153331 (+107 ms)Flushing 1588230740/ns: appending metadata at 1731423153352 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1731423153352Flushing 1588230740/table: creating writer at 1731423153373 (+21 ms)Flushing 1588230740/table: appending metadata at 1731423153391 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731423153391Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@553accff: reopening flushed file at 1731423153417 (+26 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fa4cf15: reopening flushed file at 1731423153436 (+19 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f9da39e: reopening flushed file at 1731423153455 (+19 ms)Finished flush of dataSize ~49.58 KB/50772, heapSize ~69.69 KB/71360, currentSize=0 B/0 for 1588230740 in 327ms, sequenceid=60, compaction requested=false; wal=null at 1731423153474 (+19 ms)Cleaning up temporary data from old regions at 1731423153478 (+4 ms)Running coprocessor post-open hooks at 1731423153491 (+13 ms)Region opened successfully at 1731423153492 (+1 ms) 2024-11-12T14:52:33,493 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1731423153029 2024-11-12T14:52:33,499 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=3945e1ca6f47,45845,1731422964694 2024-11-12T14:52:33,500 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3945e1ca6f47,45845,1731422964694, state=OPEN 2024-11-12T14:52:33,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:52:33,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:52:33,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:52:33,502 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=3945e1ca6f47,45845,1731422964694 2024-11-12T14:52:33,502 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:52:33,502 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:52:33,502 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:52:33,505 DEBUG [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-12T14:52:33,505 INFO [RS_OPEN_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-12T14:52:33,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-12T14:52:33,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3945e1ca6f47,45845,1731422964694 in 626 msec 2024-11-12T14:52:33,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-11-12T14:52:33,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 787 msec 2024-11-12T14:52:33,867 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:52:33,867 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,45845,1731422964694, seqNum=-1] 2024-11-12T14:52:33,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f943ce043844f379c1b7ecbf76b5a419, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:33,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=489408be5f35a96bee1ee0f68f064599, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:33,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06aa3b2e56ae4404b602d6fa98eabe33, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:33,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf0006d9b8777107f458b7008fc42ba9, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:33,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d3c298441d993e84c2d9e579914483db, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:33,919 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59fa7126e2ade1f9daaaeb8bd38b09e4, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:33,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4936dfde0b7af7ea99d36c9e0c24fefc, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:33,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f65e71cd750425cce9713e30a75c6002, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:33,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df409d6fa4df5d8fd91534d2d90eda60, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:33,920 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=92f3e9baa717d57147319661f6f00e06, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:33,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eaeef83a325d6b259b1b899f552ba8d0, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:33,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51b19d6922dec6f0774f3d761593de8f, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:33,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac225c32dbf496a52e546c7e0b1a05af, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:33,921 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7c3d3ae3b942f0414dcbb110be8bc0b, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:33,922 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb0382e4edc08a7c43f0571deb2374a4, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:33,922 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b419710b4e7b088b11da63dff9b7760e, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:33,922 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cc2e73ccc39f945b27a1d50dd2694bd, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:33,922 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a2cc208a6afe25c72a2045fb03e0ecb1, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:33,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b5e68238ca126132dbf55f0fbf38047, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:33,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d931ca9d93687bd79e96c94cb16690b, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:33,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=efc23c8cc5ebe3a5c96ef5d199223832, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:33,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b04a6d83bd2998d75a1743bdf61651e, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:33,923 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9a6f7a226796c8ff7e21a40d463cd27b, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:33,924 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa9c293d6596b8e2756cafba09b3d7dd, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:33,924 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95e5de757651e0aa099998abaa3b598f, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:33,924 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e41036448a3677ed52f58f536690bf38, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:33,924 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T14:52:33,924 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731423213924 2024-11-12T14:52:33,924 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731423273924 2024-11-12T14:52:33,924 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 1.2050 sec 2024-11-12T14:52:33,925 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 3945e1ca6f47,37185,1731422964568 had 10 regions 2024-11-12T14:52:33,925 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-12T14:52:33,925 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true 2024-11-12T14:52:33,925 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T14:52:33,926 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T14:52:34,175 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 4 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true 2024-11-12T14:52:34,175 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T14:52:34,175 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T14:52:34,175 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T14:52:34,175 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T14:52:34,176 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T14:52:34,176 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T14:52:34,176 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T14:52:34,176 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T14:52:34,177 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38303 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T14:52:34,178 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-12T14:52:34,180 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T14:52:34,180 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T14:52:34,181 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T14:52:34,181 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T14:52:34,181 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T14:52:34,181 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T14:52:34,182 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-12T14:52:34,183 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_GET_REGIONS, hasLock=false; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true 2024-11-12T14:52:34,183 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:34,183 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:34,183 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T14:52:34,183 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3945e1ca6f47,45845,1731422964694 2024-11-12T14:52:34,183 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_GET_REGIONS, hasLock=false; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T14:52:34,183 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:34,183 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:34,184 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:34,184 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3945e1ca6f47,45845,1731422964694 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:34,184 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-12T14:52:34,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=61 2024-11-12T14:52:34,190 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3945e1ca6f47,45845,1731422964694, table=hbase:meta, region=1588230740 2024-11-12T14:52:34,190 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3945e1ca6f47,45845,1731422964694, state=OPEN 2024-11-12T14:52:34,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:52:34,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:52:34,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:52:34,192 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:52:34,192 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:52:34,192 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:52:34,208 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731423184208 2024-11-12T14:52:34,208 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T14:52:34,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f943ce043844f379c1b7ecbf76b5a419, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:34,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=489408be5f35a96bee1ee0f68f064599, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:34,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06aa3b2e56ae4404b602d6fa98eabe33, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:34,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf0006d9b8777107f458b7008fc42ba9, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:34,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d3c298441d993e84c2d9e579914483db, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:34,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59fa7126e2ade1f9daaaeb8bd38b09e4, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:34,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4936dfde0b7af7ea99d36c9e0c24fefc, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:34,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f65e71cd750425cce9713e30a75c6002, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:34,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df409d6fa4df5d8fd91534d2d90eda60, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:34,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=92f3e9baa717d57147319661f6f00e06, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:34,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eaeef83a325d6b259b1b899f552ba8d0, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:34,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51b19d6922dec6f0774f3d761593de8f, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:34,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac225c32dbf496a52e546c7e0b1a05af, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:34,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7c3d3ae3b942f0414dcbb110be8bc0b, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:34,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb0382e4edc08a7c43f0571deb2374a4, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:34,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b419710b4e7b088b11da63dff9b7760e, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:34,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cc2e73ccc39f945b27a1d50dd2694bd, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:34,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a2cc208a6afe25c72a2045fb03e0ecb1, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:34,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b5e68238ca126132dbf55f0fbf38047, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:34,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d931ca9d93687bd79e96c94cb16690b, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:34,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=efc23c8cc5ebe3a5c96ef5d199223832, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:34,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b04a6d83bd2998d75a1743bdf61651e, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:34,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9a6f7a226796c8ff7e21a40d463cd27b, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:34,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa9c293d6596b8e2756cafba09b3d7dd, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:34,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95e5de757651e0aa099998abaa3b598f, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:34,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e41036448a3677ed52f58f536690bf38, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:34,223 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T14:52:34,223 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731423214223 2024-11-12T14:52:34,223 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731423274223 2024-11-12T14:52:34,223 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 14 msec 2024-11-12T14:52:34,223 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 3945e1ca6f47,37185,1731422964568 had 9 regions 2024-11-12T14:52:34,223 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-12T14:52:34,225 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true, isMeta: false 2024-11-12T14:52:34,228 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3945e1ca6f47,37185,1731422964568 WAL count=1, meta=false 2024-11-12T14:52:34,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 3945e1ca6f47%2C37185%2C1731422964568.1731422966319}] 2024-11-12T14:52:34,228 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-12T14:52:34,228 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true 2024-11-12T14:52:34,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T14:52:34,228 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T14:52:34,473 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 5 exec state=pid=57, state=WAITING:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true 2024-11-12T14:52:34,473 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T14:52:34,473 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T14:52:34,473 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T14:52:34,474 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T14:52:34,474 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T14:52:34,474 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T14:52:34,474 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T14:52:34,474 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T14:52:34,475 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T14:52:34,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T14:52:34,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T14:52:34,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T14:52:34,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T14:52:34,478 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T14:52:34,478 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T14:52:34,479 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-12T14:52:34,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=false; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true 2024-11-12T14:52:34,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:34,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:34,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T14:52:34,480 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3945e1ca6f47,45845,1731422964694 2024-11-12T14:52:34,480 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=false; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T14:52:34,480 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:34,480 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:34,480 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:34,480 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3945e1ca6f47,45845,1731422964694 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:34,480 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-12T14:52:34,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=61 2024-11-12T14:52:34,482 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3945e1ca6f47,45845,1731422964694, table=hbase:meta, region=1588230740 2024-11-12T14:52:34,483 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3945e1ca6f47,45845,1731422964694, state=OPEN 2024-11-12T14:52:34,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:52:34,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:52:34,484 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:52:34,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:52:34,485 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:52:34,485 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:52:34,485 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38303 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T14:52:34,486 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731423184486 2024-11-12T14:52:34,486 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T14:52:34,494 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f943ce043844f379c1b7ecbf76b5a419, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:34,494 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=489408be5f35a96bee1ee0f68f064599, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:34,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06aa3b2e56ae4404b602d6fa98eabe33, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:34,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf0006d9b8777107f458b7008fc42ba9, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:34,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d3c298441d993e84c2d9e579914483db, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:34,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59fa7126e2ade1f9daaaeb8bd38b09e4, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:34,495 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4936dfde0b7af7ea99d36c9e0c24fefc, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:34,496 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f65e71cd750425cce9713e30a75c6002, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:34,496 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df409d6fa4df5d8fd91534d2d90eda60, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:34,496 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=92f3e9baa717d57147319661f6f00e06, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:34,496 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eaeef83a325d6b259b1b899f552ba8d0, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:34,496 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51b19d6922dec6f0774f3d761593de8f, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:34,496 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac225c32dbf496a52e546c7e0b1a05af, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:34,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7c3d3ae3b942f0414dcbb110be8bc0b, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:34,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb0382e4edc08a7c43f0571deb2374a4, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:34,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b419710b4e7b088b11da63dff9b7760e, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:34,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cc2e73ccc39f945b27a1d50dd2694bd, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:34,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a2cc208a6afe25c72a2045fb03e0ecb1, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:34,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b5e68238ca126132dbf55f0fbf38047, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:34,497 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d931ca9d93687bd79e96c94cb16690b, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:34,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=efc23c8cc5ebe3a5c96ef5d199223832, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:34,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b04a6d83bd2998d75a1743bdf61651e, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:34,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9a6f7a226796c8ff7e21a40d463cd27b, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:34,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa9c293d6596b8e2756cafba09b3d7dd, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:34,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95e5de757651e0aa099998abaa3b598f, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:34,498 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e41036448a3677ed52f58f536690bf38, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:34,498 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T14:52:34,498 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731423214498 2024-11-12T14:52:34,498 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731423274498 2024-11-12T14:52:34,498 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-12T14:52:34,499 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true, isMeta: false 2024-11-12T14:52:34,502 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3945e1ca6f47,37185,1731422964568 WAL count=1, meta=false 2024-11-12T14:52:34,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 3945e1ca6f47%2C37185%2C1731422964568.1731422966319}] 2024-11-12T14:52:34,503 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-12T14:52:34,505 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:34,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 3945e1ca6f47%2C37185%2C1731422964568.1731422966319, worker=3945e1ca6f47,35473,1731422964757}] 2024-11-12T14:52:34,662 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59873, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T14:52:34,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35473 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-11-12T14:52:34,681 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568-splitting/3945e1ca6f47%2C37185%2C1731422964568.1731422966319, size=291.4 K (298415bytes) 2024-11-12T14:52:34,681 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568-splitting/3945e1ca6f47%2C37185%2C1731422964568.1731422966319 2024-11-12T14:52:34,681 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568-splitting/3945e1ca6f47%2C37185%2C1731422964568.1731422966319 after 0ms 2024-11-12T14:52:34,688 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568-splitting/3945e1ca6f47%2C37185%2C1731422964568.1731422966319 took 8ms 2024-11-12T14:52:34,692 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 4936dfde0b7af7ea99d36c9e0c24fefc: last_flushed_sequence_id: 18446744073709551615 2024-11-12T14:52:34,694 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 2cc2e73ccc39f945b27a1d50dd2694bd: last_flushed_sequence_id: 18446744073709551615 2024-11-12T14:52:34,696 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for d7c3d3ae3b942f0414dcbb110be8bc0b: last_flushed_sequence_id: 18446744073709551615 2024-11-12T14:52:34,698 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for d3c298441d993e84c2d9e579914483db: last_flushed_sequence_id: 18446744073709551615 2024-11-12T14:52:34,700 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for e41036448a3677ed52f58f536690bf38: last_flushed_sequence_id: 18446744073709551615 2024-11-12T14:52:34,702 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 92f3e9baa717d57147319661f6f00e06: last_flushed_sequence_id: 18446744073709551615 2024-11-12T14:52:34,703 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 9a6f7a226796c8ff7e21a40d463cd27b: last_flushed_sequence_id: 18446744073709551615 2024-11-12T14:52:34,705 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 06aa3b2e56ae4404b602d6fa98eabe33: last_flushed_sequence_id: 18446744073709551615 2024-11-12T14:52:34,706 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 2d931ca9d93687bd79e96c94cb16690b: last_flushed_sequence_id: 18446744073709551615 2024-11-12T14:52:34,724 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp 2024-11-12T14:52:34,725 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp 2024-11-12T14:52:34,733 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06aa3b2e56ae4404b602d6fa98eabe33/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp 2024-11-12T14:52:34,737 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp 2024-11-12T14:52:34,737 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp 2024-11-12T14:52:34,746 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568-splitting/3945e1ca6f47%2C37185%2C1731422964568.1731422966319 so closing down 2024-11-12T14:52:34,746 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-12T14:52:34,749 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4936dfde0b7af7ea99d36c9e0c24fefc/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp 2024-11-12T14:52:34,750 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/92f3e9baa717d57147319661f6f00e06/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp 2024-11-12T14:52:34,753 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp 2024-11-12T14:52:34,754 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp 2024-11-12T14:52:34,764 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d3c298441d993e84c2d9e579914483db/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp 2024-11-12T14:52:34,768 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp 2024-11-12T14:52:34,772 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cc2e73ccc39f945b27a1d50dd2694bd/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp 2024-11-12T14:52:34,778 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1-Writer-2 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp 2024-11-12T14:52:34,787 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2d931ca9d93687bd79e96c94cb16690b/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp 2024-11-12T14:52:34,791 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp 2024-11-12T14:52:34,809 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9a6f7a226796c8ff7e21a40d463cd27b/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp 2024-11-12T14:52:34,810 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1-Writer-2 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d7c3d3ae3b942f0414dcbb110be8bc0b/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp 2024-11-12T14:52:34,811 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e41036448a3677ed52f58f536690bf38/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp 2024-11-12T14:52:34,811 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-11-12T14:52:34,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741870_1046 (size=36767) 2024-11-12T14:52:34,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741870_1046 (size=36767) 2024-11-12T14:52:34,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741870_1046 (size=36767) 2024-11-12T14:52:34,843 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d3c298441d993e84c2d9e579914483db/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-12T14:52:34,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741869_1045 (size=262) 2024-11-12T14:52:34,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741869_1045 (size=262) 2024-11-12T14:52:34,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741869_1045 (size=262) 2024-11-12T14:52:34,845 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d3c298441d993e84c2d9e579914483db/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp to hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d3c298441d993e84c2d9e579914483db/recovered.edits/0000000000000000004 2024-11-12T14:52:34,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741871_1047 (size=36767) 2024-11-12T14:52:34,861 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/92f3e9baa717d57147319661f6f00e06/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-12T14:52:34,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741871_1047 (size=36767) 2024-11-12T14:52:34,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741871_1047 (size=36767) 2024-11-12T14:52:34,863 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/92f3e9baa717d57147319661f6f00e06/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp to hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/92f3e9baa717d57147319661f6f00e06/recovered.edits/0000000000000000004 2024-11-12T14:52:34,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741872_1048 (size=36767) 2024-11-12T14:52:34,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741872_1048 (size=36767) 2024-11-12T14:52:34,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741872_1048 (size=36767) 2024-11-12T14:52:34,880 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4936dfde0b7af7ea99d36c9e0c24fefc/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-12T14:52:34,885 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4936dfde0b7af7ea99d36c9e0c24fefc/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp to hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4936dfde0b7af7ea99d36c9e0c24fefc/recovered.edits/0000000000000000004 2024-11-12T14:52:34,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741873_1049 (size=36767) 2024-11-12T14:52:34,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741873_1049 (size=36767) 2024-11-12T14:52:34,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741873_1049 (size=36767) 2024-11-12T14:52:34,893 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2d931ca9d93687bd79e96c94cb16690b/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-12T14:52:34,895 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2d931ca9d93687bd79e96c94cb16690b/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp to hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2d931ca9d93687bd79e96c94cb16690b/recovered.edits/0000000000000000004 2024-11-12T14:52:34,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741875_1051 (size=36767) 2024-11-12T14:52:34,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741875_1051 (size=36767) 2024-11-12T14:52:34,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741875_1051 (size=36767) 2024-11-12T14:52:34,918 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06aa3b2e56ae4404b602d6fa98eabe33/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp (wrote 1 edits, skipped 0 edits in 1 ms) 2024-11-12T14:52:34,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741874_1050 (size=36767) 2024-11-12T14:52:34,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741874_1050 (size=36767) 2024-11-12T14:52:34,921 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06aa3b2e56ae4404b602d6fa98eabe33/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp to hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06aa3b2e56ae4404b602d6fa98eabe33/recovered.edits/0000000000000000004 2024-11-12T14:52:34,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741874_1050 (size=36767) 2024-11-12T14:52:34,923 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9a6f7a226796c8ff7e21a40d463cd27b/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-12T14:52:34,931 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9a6f7a226796c8ff7e21a40d463cd27b/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp to hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9a6f7a226796c8ff7e21a40d463cd27b/recovered.edits/0000000000000000004 2024-11-12T14:52:34,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741876_1052 (size=36767) 2024-11-12T14:52:34,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741876_1052 (size=36767) 2024-11-12T14:52:34,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741876_1052 (size=36767) 2024-11-12T14:52:34,954 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d7c3d3ae3b942f0414dcbb110be8bc0b/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-12T14:52:34,962 INFO [split-log-closeStream-pool-2 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d7c3d3ae3b942f0414dcbb110be8bc0b/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp to hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d7c3d3ae3b942f0414dcbb110be8bc0b/recovered.edits/0000000000000000004 2024-11-12T14:52:34,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741877_1053 (size=36767) 2024-11-12T14:52:34,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741877_1053 (size=36767) 2024-11-12T14:52:34,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741877_1053 (size=36767) 2024-11-12T14:52:34,973 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cc2e73ccc39f945b27a1d50dd2694bd/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-12T14:52:34,977 INFO [split-log-closeStream-pool-1 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cc2e73ccc39f945b27a1d50dd2694bd/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp to hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cc2e73ccc39f945b27a1d50dd2694bd/recovered.edits/0000000000000000004 2024-11-12T14:52:35,245 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e41036448a3677ed52f58f536690bf38/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-12T14:52:35,247 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e41036448a3677ed52f58f536690bf38/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp to hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e41036448a3677ed52f58f536690bf38/recovered.edits/0000000000000000004 2024-11-12T14:52:35,247 INFO [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 18 edits across 9 Regions in 559 ms; skipped=9; WAL=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568-splitting/3945e1ca6f47%2C37185%2C1731422964568.1731422966319, size=291.4 K, length=298415, corrupted=false, cancelled=false 2024-11-12T14:52:35,247 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568-splitting/3945e1ca6f47%2C37185%2C1731422964568.1731422966319, journal: Splitting hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568-splitting/3945e1ca6f47%2C37185%2C1731422964568.1731422966319, size=291.4 K (298415bytes) at 1731423154681Creating recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06aa3b2e56ae4404b602d6fa98eabe33/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp at 1731423154733 (+52 ms)Finishing writing output for hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568-splitting/3945e1ca6f47%2C37185%2C1731422964568.1731422966319 so closing down at 1731423154746 (+13 ms)Creating recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4936dfde0b7af7ea99d36c9e0c24fefc/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp at 1731423154749 (+3 ms)Creating recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/92f3e9baa717d57147319661f6f00e06/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp at 1731423154750 (+1 ms)Creating recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d3c298441d993e84c2d9e579914483db/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp at 1731423154764 (+14 ms)Creating recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cc2e73ccc39f945b27a1d50dd2694bd/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp at 1731423154772 (+8 ms)Creating recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2d931ca9d93687bd79e96c94cb16690b/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp at 1731423154787 (+15 ms)Creating recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9a6f7a226796c8ff7e21a40d463cd27b/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp at 1731423154809 (+22 ms)Creating recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d7c3d3ae3b942f0414dcbb110be8bc0b/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp at 1731423154811 (+2 ms)Creating recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e41036448a3677ed52f58f536690bf38/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp at 17314231548113 split writer threads finished at 1731423154811Closed recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d3c298441d993e84c2d9e579914483db/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731423154843 (+32 ms)Rename recovered edits hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d3c298441d993e84c2d9e579914483db/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp to hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d3c298441d993e84c2d9e579914483db/recovered.edits/0000000000000000004 at 1731423154845 (+2 ms)Closed recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/92f3e9baa717d57147319661f6f00e06/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731423154861 (+16 ms)Rename recovered edits hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/92f3e9baa717d57147319661f6f00e06/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp to hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/92f3e9baa717d57147319661f6f00e06/recovered.edits/0000000000000000004 at 1731423154864 (+3 ms)Closed recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4936dfde0b7af7ea99d36c9e0c24fefc/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731423154880 (+16 ms)Rename recovered edits hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4936dfde0b7af7ea99d36c9e0c24fefc/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp to hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4936dfde0b7af7ea99d36c9e0c24fefc/recovered.edits/0000000000000000004 at 1731423154885 (+5 ms)Closed recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2d931ca9d93687bd79e96c94cb16690b/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731423154893 (+8 ms)Rename recovered edits hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2d931ca9d93687bd79e96c94cb16690b/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp to hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2d931ca9d93687bd79e96c94cb16690b/recovered.edits/0000000000000000004 at 1731423154895 (+2 ms)Closed recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06aa3b2e56ae4404b602d6fa98eabe33/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp (wrote 1 edits, skipped 0 edits in 1 ms) at 1731423154918 (+23 ms)Rename recovered edits hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06aa3b2e56ae4404b602d6fa98eabe33/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp to hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06aa3b2e56ae4404b602d6fa98eabe33/recovered.edits/0000000000000000004 at 1731423154921 (+3 ms)Closed recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9a6f7a226796c8ff7e21a40d463cd27b/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731423154923 (+2 ms)Rename recovered edits hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9a6f7a226796c8ff7e21a40d463cd27b/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp to hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9a6f7a226796c8ff7e21a40d463cd27b/recovered.edits/0000000000000000004 at 1731423154931 (+8 ms)Closed recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d7c3d3ae3b942f0414dcbb110be8bc0b/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731423154954 (+23 ms)Rename recovered edits hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d7c3d3ae3b942f0414dcbb110be8bc0b/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp to hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d7c3d3ae3b942f0414dcbb110be8bc0b/recovered.edits/0000000000000000004 at 1731423154962 (+8 ms)Closed recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cc2e73ccc39f945b27a1d50dd2694bd/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731423154973 (+11 ms)Rename recovered edits hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cc2e73ccc39f945b27a1d50dd2694bd/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp to hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cc2e73ccc39f945b27a1d50dd2694bd/recovered.edits/0000000000000000004 at 1731423154977 (+4 ms)Closed recovered edits writer path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e41036448a3677ed52f58f536690bf38/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731423155245 (+268 ms)Rename recovered edits hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e41036448a3677ed52f58f536690bf38/recovered.edits/0000000000000000004-3945e1ca6f47%2C37185%2C1731422964568.1731422966319.temp to hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e41036448a3677ed52f58f536690bf38/recovered.edits/0000000000000000004 at 1731423155247 (+2 ms)Processed 18 edits across 9 Regions in 559 ms; skipped=9; WAL=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568-splitting/3945e1ca6f47%2C37185%2C1731422964568.1731422966319, size=291.4 K, length=298415, corrupted=false, cancelled=false at 1731423155247 2024-11-12T14:52:35,247 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568-splitting/3945e1ca6f47%2C37185%2C1731422964568.1731422966319 2024-11-12T14:52:35,247 DEBUG [RS_LOG_REPLAY_OPS-regionserver/3945e1ca6f47:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-12T14:52:35,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38303 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-12T14:52:35,251 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568-splitting/3945e1ca6f47%2C37185%2C1731422964568.1731422966319 to hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/oldWALs 2024-11-12T14:52:35,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-12T14:52:35,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3945e1ca6f47%2C37185%2C1731422964568.1731422966319, worker=3945e1ca6f47,35473,1731422964757 in 745 msec 2024-11-12T14:52:35,256 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:35,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-12T14:52:35,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3945e1ca6f47%2C37185%2C1731422964568.1731422966319, worker=3945e1ca6f47,35473,1731422964757 in 754 msec 2024-11-12T14:52:35,261 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568-splitting dir is empty, no logs to split. 2024-11-12T14:52:35,261 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3945e1ca6f47,37185,1731422964568 WAL count=0, meta=false 2024-11-12T14:52:35,261 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 3945e1ca6f47,37185,1731422964568 WAL splitting is done? wals=0, meta=false 2024-11-12T14:52:35,262 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-12T14:52:35,262 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true 2024-11-12T14:52:35,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T14:52:35,263 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T14:52:35,499 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 6 exec state=pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true 2024-11-12T14:52:35,499 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T14:52:35,499 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T14:52:35,499 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T14:52:35,499 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T14:52:35,500 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T14:52:35,500 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T14:52:35,500 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T14:52:35,500 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T14:52:35,501 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T14:52:35,504 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T14:52:35,504 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T14:52:35,504 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T14:52:35,504 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T14:52:35,504 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T14:52:35,504 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T14:52:35,504 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T14:52:35,504 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T14:52:35,505 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-12T14:52:35,506 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=false; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true 2024-11-12T14:52:35,506 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:35,506 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:35,506 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T14:52:35,506 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3945e1ca6f47,45845,1731422964694 2024-11-12T14:52:35,507 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3945e1ca6f47%2C37185%2C1731422964568.1731422966319, worker=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:35,507 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3945e1ca6f47%2C37185%2C1731422964568.1731422966319, worker=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:35,507 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_DELETE_SPLIT_WALS_DIR, hasLock=false; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T14:52:35,507 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:35,507 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:35,507 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:35,507 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3945e1ca6f47,45845,1731422964694 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:35,507 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3945e1ca6f47%2C37185%2C1731422964568.1731422966319, worker=3945e1ca6f47,35473,1731422964757 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:35,507 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3945e1ca6f47%2C37185%2C1731422964568.1731422966319, worker=3945e1ca6f47,35473,1731422964757 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:35,507 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-12T14:52:35,508 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38303 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T14:52:35,509 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=61 2024-11-12T14:52:35,509 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3945e1ca6f47,45845,1731422964694, table=hbase:meta, region=1588230740 2024-11-12T14:52:35,509 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3945e1ca6f47,45845,1731422964694, state=OPEN 2024-11-12T14:52:35,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:52:35,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:52:35,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:52:35,511 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:52:35,511 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:52:35,514 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:52:35,518 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731423185518 2024-11-12T14:52:35,518 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T14:52:35,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f943ce043844f379c1b7ecbf76b5a419, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:35,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=489408be5f35a96bee1ee0f68f064599, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:35,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06aa3b2e56ae4404b602d6fa98eabe33, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:35,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf0006d9b8777107f458b7008fc42ba9, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:35,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d3c298441d993e84c2d9e579914483db, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:35,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59fa7126e2ade1f9daaaeb8bd38b09e4, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:35,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4936dfde0b7af7ea99d36c9e0c24fefc, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:35,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f65e71cd750425cce9713e30a75c6002, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:35,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df409d6fa4df5d8fd91534d2d90eda60, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:35,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=92f3e9baa717d57147319661f6f00e06, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:35,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eaeef83a325d6b259b1b899f552ba8d0, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:35,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51b19d6922dec6f0774f3d761593de8f, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:35,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac225c32dbf496a52e546c7e0b1a05af, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:35,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7c3d3ae3b942f0414dcbb110be8bc0b, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:35,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb0382e4edc08a7c43f0571deb2374a4, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:35,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b419710b4e7b088b11da63dff9b7760e, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:35,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cc2e73ccc39f945b27a1d50dd2694bd, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:35,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a2cc208a6afe25c72a2045fb03e0ecb1, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:35,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b5e68238ca126132dbf55f0fbf38047, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:35,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d931ca9d93687bd79e96c94cb16690b, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:35,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=efc23c8cc5ebe3a5c96ef5d199223832, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:35,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b04a6d83bd2998d75a1743bdf61651e, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:35,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9a6f7a226796c8ff7e21a40d463cd27b, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:35,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa9c293d6596b8e2756cafba09b3d7dd, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:35,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95e5de757651e0aa099998abaa3b598f, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:35,527 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e41036448a3677ed52f58f536690bf38, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:35,527 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T14:52:35,527 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731423215527 2024-11-12T14:52:35,527 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731423275527 2024-11-12T14:52:35,527 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-12T14:52:35,528 INFO [PEWorker-1 {}] master.MasterWalManager(331): Log dir for server 3945e1ca6f47,37185,1731422964568 does not exist 2024-11-12T14:52:35,528 INFO [PEWorker-1 {}] master.SplitWALManager(105): 3945e1ca6f47,37185,1731422964568 WAL count=0, meta=false 2024-11-12T14:52:35,528 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 3945e1ca6f47,37185,1731422964568 WAL splitting is done? wals=0, meta=false 2024-11-12T14:52:35,529 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 3945e1ca6f47,37185,1731422964568 failed, ignore...File hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,37185,1731422964568-splitting does not exist. 2024-11-12T14:52:35,529 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-12T14:52:35,531 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e41036448a3677ed52f58f536690bf38, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2cc2e73ccc39f945b27a1d50dd2694bd, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9a6f7a226796c8ff7e21a40d463cd27b, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4936dfde0b7af7ea99d36c9e0c24fefc, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=92f3e9baa717d57147319661f6f00e06, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2d931ca9d93687bd79e96c94cb16690b, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=06aa3b2e56ae4404b602d6fa98eabe33, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d3c298441d993e84c2d9e579914483db, ASSIGN}, {pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d7c3d3ae3b942f0414dcbb110be8bc0b, ASSIGN}] 2024-11-12T14:52:35,531 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-12T14:52:35,531 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_FINISH, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true 2024-11-12T14:52:35,531 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T14:52:35,531 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=WAITING:SERVER_CRASH_FINISH, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T14:52:35,777 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 7 exec state=pid=57, state=WAITING:SERVER_CRASH_FINISH, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true 2024-11-12T14:52:35,777 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T14:52:35,777 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T14:52:35,777 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T14:52:35,777 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T14:52:35,778 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T14:52:35,778 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T14:52:35,778 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T14:52:35,778 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T14:52:35,779 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-12T14:52:35,781 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T14:52:35,781 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T14:52:35,782 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T14:52:35,782 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T14:52:35,782 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T14:52:35,782 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T14:52:35,782 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T14:52:35,782 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T14:52:35,783 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-12T14:52:35,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=false; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true 2024-11-12T14:52:35,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:35,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:35,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T14:52:35,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3945e1ca6f47,45845,1731422964694 2024-11-12T14:52:35,786 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3945e1ca6f47%2C37185%2C1731422964568.1731422966319, worker=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:35,786 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3945e1ca6f47%2C37185%2C1731422964568.1731422966319, worker=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:35,786 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_ASSIGN, hasLock=false; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T14:52:35,786 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:35,786 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:35,786 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:35,786 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3945e1ca6f47,45845,1731422964694 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:35,786 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3945e1ca6f47%2C37185%2C1731422964568.1731422966319, worker=3945e1ca6f47,35473,1731422964757 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:35,786 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3945e1ca6f47%2C37185%2C1731422964568.1731422966319, worker=3945e1ca6f47,35473,1731422964757 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:35,786 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-12T14:52:35,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=61 2024-11-12T14:52:35,787 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3945e1ca6f47,45845,1731422964694, table=hbase:meta, region=1588230740 2024-11-12T14:52:35,787 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3945e1ca6f47,45845,1731422964694, state=OPEN 2024-11-12T14:52:35,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:52:35,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:52:35,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:52:35,789 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:52:35,789 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:52:35,789 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:52:35,797 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731423185797 2024-11-12T14:52:35,797 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T14:52:35,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f943ce043844f379c1b7ecbf76b5a419, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:35,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=489408be5f35a96bee1ee0f68f064599, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:35,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06aa3b2e56ae4404b602d6fa98eabe33, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:35,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf0006d9b8777107f458b7008fc42ba9, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:35,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d3c298441d993e84c2d9e579914483db, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:35,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59fa7126e2ade1f9daaaeb8bd38b09e4, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:35,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4936dfde0b7af7ea99d36c9e0c24fefc, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:35,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f65e71cd750425cce9713e30a75c6002, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:35,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df409d6fa4df5d8fd91534d2d90eda60, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:35,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=92f3e9baa717d57147319661f6f00e06, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:35,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eaeef83a325d6b259b1b899f552ba8d0, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:35,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51b19d6922dec6f0774f3d761593de8f, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:35,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac225c32dbf496a52e546c7e0b1a05af, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:35,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7c3d3ae3b942f0414dcbb110be8bc0b, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:35,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb0382e4edc08a7c43f0571deb2374a4, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:35,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b419710b4e7b088b11da63dff9b7760e, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:35,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cc2e73ccc39f945b27a1d50dd2694bd, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:35,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a2cc208a6afe25c72a2045fb03e0ecb1, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:35,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b5e68238ca126132dbf55f0fbf38047, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:35,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d931ca9d93687bd79e96c94cb16690b, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:35,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=efc23c8cc5ebe3a5c96ef5d199223832, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:35,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b04a6d83bd2998d75a1743bdf61651e, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:35,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9a6f7a226796c8ff7e21a40d463cd27b, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:35,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa9c293d6596b8e2756cafba09b3d7dd, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:35,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95e5de757651e0aa099998abaa3b598f, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:35,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e41036448a3677ed52f58f536690bf38, regionState=OPEN, lastHost=3945e1ca6f47,37185,1731422964568, regionLocation=3945e1ca6f47,37185,1731422964568, openSeqNum=2 2024-11-12T14:52:35,807 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T14:52:35,807 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731423215807 2024-11-12T14:52:35,807 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731423275807 2024-11-12T14:52:35,807 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-12T14:52:35,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e41036448a3677ed52f58f536690bf38, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2cc2e73ccc39f945b27a1d50dd2694bd, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9a6f7a226796c8ff7e21a40d463cd27b, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4936dfde0b7af7ea99d36c9e0c24fefc, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=92f3e9baa717d57147319661f6f00e06, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2d931ca9d93687bd79e96c94cb16690b, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=06aa3b2e56ae4404b602d6fa98eabe33, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d3c298441d993e84c2d9e579914483db, ASSIGN}, {pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d7c3d3ae3b942f0414dcbb110be8bc0b, ASSIGN}] 2024-11-12T14:52:35,808 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-12T14:52:35,810 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d7c3d3ae3b942f0414dcbb110be8bc0b, ASSIGN 2024-11-12T14:52:35,812 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d7c3d3ae3b942f0414dcbb110be8bc0b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T14:52:35,813 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d3c298441d993e84c2d9e579914483db, ASSIGN 2024-11-12T14:52:35,815 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d3c298441d993e84c2d9e579914483db, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T14:52:35,816 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=06aa3b2e56ae4404b602d6fa98eabe33, ASSIGN 2024-11-12T14:52:35,817 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=06aa3b2e56ae4404b602d6fa98eabe33, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T14:52:35,817 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2d931ca9d93687bd79e96c94cb16690b, ASSIGN 2024-11-12T14:52:35,818 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2d931ca9d93687bd79e96c94cb16690b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T14:52:35,819 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=92f3e9baa717d57147319661f6f00e06, ASSIGN 2024-11-12T14:52:35,820 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=92f3e9baa717d57147319661f6f00e06, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T14:52:35,820 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4936dfde0b7af7ea99d36c9e0c24fefc, ASSIGN 2024-11-12T14:52:35,821 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4936dfde0b7af7ea99d36c9e0c24fefc, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T14:52:35,822 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9a6f7a226796c8ff7e21a40d463cd27b, ASSIGN 2024-11-12T14:52:35,823 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9a6f7a226796c8ff7e21a40d463cd27b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T14:52:35,824 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2cc2e73ccc39f945b27a1d50dd2694bd, ASSIGN 2024-11-12T14:52:35,825 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2cc2e73ccc39f945b27a1d50dd2694bd, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T14:52:35,826 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e41036448a3677ed52f58f536690bf38, ASSIGN 2024-11-12T14:52:35,827 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e41036448a3677ed52f58f536690bf38, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-12T14:52:35,963 DEBUG [3945e1ca6f47:38303 {}] balancer.BalancerClusterState(204): Hosts are {3945e1ca6f47=0} racks are {/default-rack=0} 2024-11-12T14:52:35,963 DEBUG [3945e1ca6f47:38303 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-12T14:52:35,963 DEBUG [3945e1ca6f47:38303 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-12T14:52:35,963 DEBUG [3945e1ca6f47:38303 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-12T14:52:35,963 DEBUG [3945e1ca6f47:38303 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-12T14:52:35,963 INFO [3945e1ca6f47:38303 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-12T14:52:35,963 INFO [3945e1ca6f47:38303 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-12T14:52:35,963 DEBUG [3945e1ca6f47:38303 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-12T14:52:35,964 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=e41036448a3677ed52f58f536690bf38, regionState=OPENING, regionLocation=3945e1ca6f47,45845,1731422964694 2024-11-12T14:52:35,965 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=2d931ca9d93687bd79e96c94cb16690b, regionState=OPENING, regionLocation=3945e1ca6f47,45845,1731422964694 2024-11-12T14:52:35,965 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=d7c3d3ae3b942f0414dcbb110be8bc0b, regionState=OPENING, regionLocation=3945e1ca6f47,45845,1731422964694 2024-11-12T14:52:35,966 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=4936dfde0b7af7ea99d36c9e0c24fefc, regionState=OPENING, regionLocation=3945e1ca6f47,45845,1731422964694 2024-11-12T14:52:35,966 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=06aa3b2e56ae4404b602d6fa98eabe33, regionState=OPENING, regionLocation=3945e1ca6f47,45845,1731422964694 2024-11-12T14:52:35,967 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=9a6f7a226796c8ff7e21a40d463cd27b, regionState=OPENING, regionLocation=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:35,967 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e41036448a3677ed52f58f536690bf38, ASSIGN because future has completed 2024-11-12T14:52:35,967 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=2cc2e73ccc39f945b27a1d50dd2694bd, regionState=OPENING, regionLocation=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:35,968 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=92f3e9baa717d57147319661f6f00e06, regionState=OPENING, regionLocation=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:35,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure e41036448a3677ed52f58f536690bf38, server=3945e1ca6f47,45845,1731422964694}] 2024-11-12T14:52:35,970 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2d931ca9d93687bd79e96c94cb16690b, ASSIGN because future has completed 2024-11-12T14:52:35,974 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d7c3d3ae3b942f0414dcbb110be8bc0b, ASSIGN because future has completed 2024-11-12T14:52:35,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4936dfde0b7af7ea99d36c9e0c24fefc, ASSIGN because future has completed 2024-11-12T14:52:35,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4936dfde0b7af7ea99d36c9e0c24fefc, server=3945e1ca6f47,45845,1731422964694}] 2024-11-12T14:52:35,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=06aa3b2e56ae4404b602d6fa98eabe33, ASSIGN because future has completed 2024-11-12T14:52:35,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9a6f7a226796c8ff7e21a40d463cd27b, ASSIGN because future has completed 2024-11-12T14:52:35,986 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2cc2e73ccc39f945b27a1d50dd2694bd, ASSIGN because future has completed 2024-11-12T14:52:35,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=92f3e9baa717d57147319661f6f00e06, ASSIGN because future has completed 2024-11-12T14:52:35,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92f3e9baa717d57147319661f6f00e06, server=3945e1ca6f47,35473,1731422964757}] 2024-11-12T14:52:35,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2cc2e73ccc39f945b27a1d50dd2694bd, server=3945e1ca6f47,35473,1731422964757}] 2024-11-12T14:52:35,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9a6f7a226796c8ff7e21a40d463cd27b, server=3945e1ca6f47,35473,1731422964757}] 2024-11-12T14:52:35,997 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06aa3b2e56ae4404b602d6fa98eabe33, server=3945e1ca6f47,45845,1731422964694}] 2024-11-12T14:52:36,000 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure d7c3d3ae3b942f0414dcbb110be8bc0b, server=3945e1ca6f47,45845,1731422964694}] 2024-11-12T14:52:36,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d931ca9d93687bd79e96c94cb16690b, server=3945e1ca6f47,45845,1731422964694}] 2024-11-12T14:52:36,004 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=d3c298441d993e84c2d9e579914483db, regionState=OPENING, regionLocation=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:36,006 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d3c298441d993e84c2d9e579914483db, ASSIGN because future has completed 2024-11-12T14:52:36,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3c298441d993e84c2d9e579914483db, server=3945e1ca6f47,35473,1731422964757}] 2024-11-12T14:52:36,129 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc. 2024-11-12T14:52:36,129 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => 4936dfde0b7af7ea99d36c9e0c24fefc, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-12T14:52:36,130 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 4936dfde0b7af7ea99d36c9e0c24fefc 2024-11-12T14:52:36,130 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:52:36,130 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for 4936dfde0b7af7ea99d36c9e0c24fefc 2024-11-12T14:52:36,130 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for 4936dfde0b7af7ea99d36c9e0c24fefc 2024-11-12T14:52:36,132 INFO [StoreOpener-4936dfde0b7af7ea99d36c9e0c24fefc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 4936dfde0b7af7ea99d36c9e0c24fefc 2024-11-12T14:52:36,132 INFO [StoreOpener-4936dfde0b7af7ea99d36c9e0c24fefc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4936dfde0b7af7ea99d36c9e0c24fefc columnFamilyName colfamily11 2024-11-12T14:52:36,133 DEBUG [StoreOpener-4936dfde0b7af7ea99d36c9e0c24fefc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:36,133 INFO [StoreOpener-4936dfde0b7af7ea99d36c9e0c24fefc-1 {}] regionserver.HStore(327): Store=4936dfde0b7af7ea99d36c9e0c24fefc/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:52:36,133 INFO [StoreOpener-4936dfde0b7af7ea99d36c9e0c24fefc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 4936dfde0b7af7ea99d36c9e0c24fefc 2024-11-12T14:52:36,134 INFO [StoreOpener-4936dfde0b7af7ea99d36c9e0c24fefc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4936dfde0b7af7ea99d36c9e0c24fefc columnFamilyName colfamily21 2024-11-12T14:52:36,135 DEBUG [StoreOpener-4936dfde0b7af7ea99d36c9e0c24fefc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:36,135 INFO [StoreOpener-4936dfde0b7af7ea99d36c9e0c24fefc-1 {}] regionserver.HStore(327): Store=4936dfde0b7af7ea99d36c9e0c24fefc/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:52:36,135 INFO [StoreOpener-4936dfde0b7af7ea99d36c9e0c24fefc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 4936dfde0b7af7ea99d36c9e0c24fefc 2024-11-12T14:52:36,136 INFO [StoreOpener-4936dfde0b7af7ea99d36c9e0c24fefc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4936dfde0b7af7ea99d36c9e0c24fefc columnFamilyName colfamily31 2024-11-12T14:52:36,136 DEBUG [StoreOpener-4936dfde0b7af7ea99d36c9e0c24fefc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:36,137 INFO [StoreOpener-4936dfde0b7af7ea99d36c9e0c24fefc-1 {}] regionserver.HStore(327): Store=4936dfde0b7af7ea99d36c9e0c24fefc/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:52:36,137 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for 4936dfde0b7af7ea99d36c9e0c24fefc 2024-11-12T14:52:36,138 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4936dfde0b7af7ea99d36c9e0c24fefc 2024-11-12T14:52:36,140 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4936dfde0b7af7ea99d36c9e0c24fefc 2024-11-12T14:52:36,141 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4936dfde0b7af7ea99d36c9e0c24fefc/recovered.edits/0000000000000000004 2024-11-12T14:52:36,145 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db. 2024-11-12T14:52:36,145 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => d3c298441d993e84c2d9e579914483db, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-12T14:52:36,146 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true d3c298441d993e84c2d9e579914483db 2024-11-12T14:52:36,146 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:52:36,146 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for d3c298441d993e84c2d9e579914483db 2024-11-12T14:52:36,146 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for d3c298441d993e84c2d9e579914483db 2024-11-12T14:52:36,147 INFO [StoreOpener-d3c298441d993e84c2d9e579914483db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region d3c298441d993e84c2d9e579914483db 2024-11-12T14:52:36,148 INFO [StoreOpener-d3c298441d993e84c2d9e579914483db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3c298441d993e84c2d9e579914483db columnFamilyName colfamily11 2024-11-12T14:52:36,149 DEBUG [StoreOpener-d3c298441d993e84c2d9e579914483db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:36,149 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4936dfde0b7af7ea99d36c9e0c24fefc/recovered.edits/0000000000000000004 2024-11-12T14:52:36,149 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(2902): Flushing 4936dfde0b7af7ea99d36c9e0c24fefc 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-12T14:52:36,149 INFO [StoreOpener-d3c298441d993e84c2d9e579914483db-1 {}] regionserver.HStore(327): Store=d3c298441d993e84c2d9e579914483db/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:52:36,149 INFO [StoreOpener-d3c298441d993e84c2d9e579914483db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region d3c298441d993e84c2d9e579914483db 2024-11-12T14:52:36,151 INFO [StoreOpener-d3c298441d993e84c2d9e579914483db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3c298441d993e84c2d9e579914483db columnFamilyName colfamily21 2024-11-12T14:52:36,151 DEBUG [StoreOpener-d3c298441d993e84c2d9e579914483db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:36,151 INFO [StoreOpener-d3c298441d993e84c2d9e579914483db-1 {}] regionserver.HStore(327): Store=d3c298441d993e84c2d9e579914483db/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:52:36,152 INFO [StoreOpener-d3c298441d993e84c2d9e579914483db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region d3c298441d993e84c2d9e579914483db 2024-11-12T14:52:36,153 INFO [StoreOpener-d3c298441d993e84c2d9e579914483db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d3c298441d993e84c2d9e579914483db columnFamilyName colfamily31 2024-11-12T14:52:36,153 DEBUG [StoreOpener-d3c298441d993e84c2d9e579914483db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:36,153 INFO [StoreOpener-d3c298441d993e84c2d9e579914483db-1 {}] regionserver.HStore(327): Store=d3c298441d993e84c2d9e579914483db/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:52:36,154 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for d3c298441d993e84c2d9e579914483db 2024-11-12T14:52:36,155 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d3c298441d993e84c2d9e579914483db 2024-11-12T14:52:36,158 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d3c298441d993e84c2d9e579914483db 2024-11-12T14:52:36,160 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d3c298441d993e84c2d9e579914483db/recovered.edits/0000000000000000004 2024-11-12T14:52:36,174 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d3c298441d993e84c2d9e579914483db/recovered.edits/0000000000000000004 2024-11-12T14:52:36,174 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(2902): Flushing d3c298441d993e84c2d9e579914483db 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-12T14:52:36,182 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4936dfde0b7af7ea99d36c9e0c24fefc/.tmp/colfamily11/09d15db8d2294d449a3f9e767368a9f4 is 52, key is ggg/colfamily11:colfamily11/1731422971526/Put/seqid=0 2024-11-12T14:52:36,200 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d3c298441d993e84c2d9e579914483db/.tmp/colfamily11/189be41ddd7e41659c5e6a7cdf192775 is 52, key is eee/colfamily11:colfamily11/1731422971448/Put/seqid=0 2024-11-12T14:52:36,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741878_1054 (size=39263) 2024-11-12T14:52:36,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741878_1054 (size=39263) 2024-11-12T14:52:36,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741878_1054 (size=39263) 2024-11-12T14:52:36,228 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4936dfde0b7af7ea99d36c9e0c24fefc/.tmp/colfamily11/09d15db8d2294d449a3f9e767368a9f4 2024-11-12T14:52:36,240 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4936dfde0b7af7ea99d36c9e0c24fefc/.tmp/colfamily11/09d15db8d2294d449a3f9e767368a9f4 as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4936dfde0b7af7ea99d36c9e0c24fefc/colfamily11/09d15db8d2294d449a3f9e767368a9f4 2024-11-12T14:52:36,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741879_1055 (size=39263) 2024-11-12T14:52:36,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741879_1055 (size=39263) 2024-11-12T14:52:36,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741879_1055 (size=39263) 2024-11-12T14:52:36,249 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d3c298441d993e84c2d9e579914483db/.tmp/colfamily11/189be41ddd7e41659c5e6a7cdf192775 2024-11-12T14:52:36,254 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4936dfde0b7af7ea99d36c9e0c24fefc/colfamily11/09d15db8d2294d449a3f9e767368a9f4, entries=703, sequenceid=4, filesize=38.3 K 2024-11-12T14:52:36,254 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 4936dfde0b7af7ea99d36c9e0c24fefc in 105ms, sequenceid=4, compaction requested=false; wal=null 2024-11-12T14:52:36,257 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4936dfde0b7af7ea99d36c9e0c24fefc/recovered.edits/0000000000000000004 2024-11-12T14:52:36,258 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for 4936dfde0b7af7ea99d36c9e0c24fefc 2024-11-12T14:52:36,258 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for 4936dfde0b7af7ea99d36c9e0c24fefc 2024-11-12T14:52:36,260 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:52:36,262 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d3c298441d993e84c2d9e579914483db/.tmp/colfamily11/189be41ddd7e41659c5e6a7cdf192775 as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d3c298441d993e84c2d9e579914483db/colfamily11/189be41ddd7e41659c5e6a7cdf192775 2024-11-12T14:52:36,263 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for 4936dfde0b7af7ea99d36c9e0c24fefc 2024-11-12T14:52:36,266 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4936dfde0b7af7ea99d36c9e0c24fefc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T14:52:36,267 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened 4936dfde0b7af7ea99d36c9e0c24fefc; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59480183, jitterRate=-0.11367620527744293}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:52:36,267 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4936dfde0b7af7ea99d36c9e0c24fefc 2024-11-12T14:52:36,267 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for 4936dfde0b7af7ea99d36c9e0c24fefc: Running coprocessor pre-open hook at 1731423156130Writing region info on filesystem at 1731423156130Initializing all the Stores at 1731423156131 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423156131Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423156131Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423156131Obtaining lock to block concurrent updates at 1731423156149 (+18 ms)Preparing flush snapshotting stores in 4936dfde0b7af7ea99d36c9e0c24fefc at 1731423156149Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731423156149Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc. at 1731423156149Flushing 4936dfde0b7af7ea99d36c9e0c24fefc/colfamily11: creating writer at 1731423156149Flushing 4936dfde0b7af7ea99d36c9e0c24fefc/colfamily11: appending metadata at 1731423156181 (+32 ms)Flushing 4936dfde0b7af7ea99d36c9e0c24fefc/colfamily11: closing flushed file at 1731423156181Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a94a7ba: reopening flushed file at 1731423156238 (+57 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 4936dfde0b7af7ea99d36c9e0c24fefc in 105ms, sequenceid=4, compaction requested=false; wal=null at 1731423156254 (+16 ms)Cleaning up temporary data from old regions at 1731423156259 (+5 ms)Running coprocessor post-open hooks at 1731423156267 (+8 ms)Region opened successfully at 1731423156267 2024-11-12T14:52:36,268 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc., pid=74, masterSystemTime=1731423156124 2024-11-12T14:52:36,271 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d3c298441d993e84c2d9e579914483db/colfamily11/189be41ddd7e41659c5e6a7cdf192775, entries=703, sequenceid=4, filesize=38.3 K 2024-11-12T14:52:36,271 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for d3c298441d993e84c2d9e579914483db in 97ms, sequenceid=4, compaction requested=false; wal=null 2024-11-12T14:52:36,271 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=4936dfde0b7af7ea99d36c9e0c24fefc, regionState=OPEN, openSeqNum=5, regionLocation=3945e1ca6f47,45845,1731422964694 2024-11-12T14:52:36,273 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc. 2024-11-12T14:52:36,273 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc. 2024-11-12T14:52:36,273 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38. 2024-11-12T14:52:36,273 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => e41036448a3677ed52f58f536690bf38, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-12T14:52:36,274 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d3c298441d993e84c2d9e579914483db/recovered.edits/0000000000000000004 2024-11-12T14:52:36,274 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true e41036448a3677ed52f58f536690bf38 2024-11-12T14:52:36,274 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:52:36,274 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4936dfde0b7af7ea99d36c9e0c24fefc, server=3945e1ca6f47,45845,1731422964694 because future has completed 2024-11-12T14:52:36,274 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for e41036448a3677ed52f58f536690bf38 2024-11-12T14:52:36,274 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for e41036448a3677ed52f58f536690bf38 2024-11-12T14:52:36,275 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for d3c298441d993e84c2d9e579914483db 2024-11-12T14:52:36,275 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for d3c298441d993e84c2d9e579914483db 2024-11-12T14:52:36,276 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:52:36,279 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=67 2024-11-12T14:52:36,279 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure 4936dfde0b7af7ea99d36c9e0c24fefc, server=3945e1ca6f47,45845,1731422964694 in 298 msec 2024-11-12T14:52:36,280 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for d3c298441d993e84c2d9e579914483db 2024-11-12T14:52:36,281 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4936dfde0b7af7ea99d36c9e0c24fefc, ASSIGN in 471 msec 2024-11-12T14:52:36,285 INFO [StoreOpener-e41036448a3677ed52f58f536690bf38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region e41036448a3677ed52f58f536690bf38 2024-11-12T14:52:36,287 INFO [StoreOpener-e41036448a3677ed52f58f536690bf38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e41036448a3677ed52f58f536690bf38 columnFamilyName colfamily11 2024-11-12T14:52:36,287 DEBUG [StoreOpener-e41036448a3677ed52f58f536690bf38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:36,287 INFO [StoreOpener-e41036448a3677ed52f58f536690bf38-1 {}] regionserver.HStore(327): Store=e41036448a3677ed52f58f536690bf38/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:52:36,288 INFO [StoreOpener-e41036448a3677ed52f58f536690bf38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region e41036448a3677ed52f58f536690bf38 2024-11-12T14:52:36,289 INFO [StoreOpener-e41036448a3677ed52f58f536690bf38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e41036448a3677ed52f58f536690bf38 columnFamilyName colfamily21 2024-11-12T14:52:36,289 DEBUG [StoreOpener-e41036448a3677ed52f58f536690bf38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:36,290 INFO [StoreOpener-e41036448a3677ed52f58f536690bf38-1 {}] regionserver.HStore(327): Store=e41036448a3677ed52f58f536690bf38/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:52:36,290 INFO [StoreOpener-e41036448a3677ed52f58f536690bf38-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region e41036448a3677ed52f58f536690bf38 2024-11-12T14:52:36,290 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d3c298441d993e84c2d9e579914483db/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T14:52:36,291 INFO [StoreOpener-e41036448a3677ed52f58f536690bf38-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e41036448a3677ed52f58f536690bf38 columnFamilyName colfamily31 2024-11-12T14:52:36,291 DEBUG [StoreOpener-e41036448a3677ed52f58f536690bf38-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:36,292 INFO [StoreOpener-e41036448a3677ed52f58f536690bf38-1 {}] regionserver.HStore(327): Store=e41036448a3677ed52f58f536690bf38/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:52:36,292 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened d3c298441d993e84c2d9e579914483db; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74855641, jitterRate=0.11543597280979156}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:52:36,292 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d3c298441d993e84c2d9e579914483db 2024-11-12T14:52:36,292 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for e41036448a3677ed52f58f536690bf38 2024-11-12T14:52:36,292 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for d3c298441d993e84c2d9e579914483db: Running coprocessor pre-open hook at 1731423156146Writing region info on filesystem at 1731423156146Initializing all the Stores at 1731423156147 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423156147Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423156147Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423156147Obtaining lock to block concurrent updates at 1731423156174 (+27 ms)Preparing flush snapshotting stores in d3c298441d993e84c2d9e579914483db at 1731423156174Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731423156175 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db. at 1731423156175Flushing d3c298441d993e84c2d9e579914483db/colfamily11: creating writer at 1731423156175Flushing d3c298441d993e84c2d9e579914483db/colfamily11: appending metadata at 1731423156200 (+25 ms)Flushing d3c298441d993e84c2d9e579914483db/colfamily11: closing flushed file at 1731423156200Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5773f9f7: reopening flushed file at 1731423156260 (+60 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for d3c298441d993e84c2d9e579914483db in 97ms, sequenceid=4, compaction requested=false; wal=null at 1731423156271 (+11 ms)Cleaning up temporary data from old regions at 1731423156275 (+4 ms)Running coprocessor post-open hooks at 1731423156292 (+17 ms)Region opened successfully at 1731423156292 2024-11-12T14:52:36,293 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db., pid=81, masterSystemTime=1731423156141 2024-11-12T14:52:36,297 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db. 2024-11-12T14:52:36,297 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db. 2024-11-12T14:52:36,297 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06. 2024-11-12T14:52:36,297 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=d3c298441d993e84c2d9e579914483db, regionState=OPEN, openSeqNum=5, regionLocation=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:36,297 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 92f3e9baa717d57147319661f6f00e06, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-12T14:52:36,298 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 92f3e9baa717d57147319661f6f00e06 2024-11-12T14:52:36,298 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:52:36,298 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 92f3e9baa717d57147319661f6f00e06 2024-11-12T14:52:36,298 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 92f3e9baa717d57147319661f6f00e06 2024-11-12T14:52:36,300 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure d3c298441d993e84c2d9e579914483db, server=3945e1ca6f47,35473,1731422964757 because future has completed 2024-11-12T14:52:36,301 INFO [StoreOpener-92f3e9baa717d57147319661f6f00e06-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 92f3e9baa717d57147319661f6f00e06 2024-11-12T14:52:36,302 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e41036448a3677ed52f58f536690bf38 2024-11-12T14:52:36,306 INFO [StoreOpener-92f3e9baa717d57147319661f6f00e06-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92f3e9baa717d57147319661f6f00e06 columnFamilyName colfamily11 2024-11-12T14:52:36,306 DEBUG [StoreOpener-92f3e9baa717d57147319661f6f00e06-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:36,311 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e41036448a3677ed52f58f536690bf38 2024-11-12T14:52:36,312 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e41036448a3677ed52f58f536690bf38/recovered.edits/0000000000000000004 2024-11-12T14:52:36,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=71 2024-11-12T14:52:36,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=71, state=SUCCESS, hasLock=false; OpenRegionProcedure d3c298441d993e84c2d9e579914483db, server=3945e1ca6f47,35473,1731422964757 in 294 msec 2024-11-12T14:52:36,316 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e41036448a3677ed52f58f536690bf38/recovered.edits/0000000000000000004 2024-11-12T14:52:36,316 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(2902): Flushing e41036448a3677ed52f58f536690bf38 3/3 column families, dataSize=48 B heapSize=888 B 2024-11-12T14:52:36,316 INFO [StoreOpener-92f3e9baa717d57147319661f6f00e06-1 {}] regionserver.HStore(327): Store=92f3e9baa717d57147319661f6f00e06/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:52:36,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d3c298441d993e84c2d9e579914483db, ASSIGN in 507 msec 2024-11-12T14:52:36,316 INFO [StoreOpener-92f3e9baa717d57147319661f6f00e06-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 92f3e9baa717d57147319661f6f00e06 2024-11-12T14:52:36,320 INFO [StoreOpener-92f3e9baa717d57147319661f6f00e06-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92f3e9baa717d57147319661f6f00e06 columnFamilyName colfamily21 2024-11-12T14:52:36,320 DEBUG [StoreOpener-92f3e9baa717d57147319661f6f00e06-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:36,321 INFO [StoreOpener-92f3e9baa717d57147319661f6f00e06-1 {}] regionserver.HStore(327): Store=92f3e9baa717d57147319661f6f00e06/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:52:36,321 INFO [StoreOpener-92f3e9baa717d57147319661f6f00e06-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 92f3e9baa717d57147319661f6f00e06 2024-11-12T14:52:36,322 INFO [StoreOpener-92f3e9baa717d57147319661f6f00e06-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 92f3e9baa717d57147319661f6f00e06 columnFamilyName colfamily31 2024-11-12T14:52:36,322 DEBUG [StoreOpener-92f3e9baa717d57147319661f6f00e06-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:36,323 INFO [StoreOpener-92f3e9baa717d57147319661f6f00e06-1 {}] regionserver.HStore(327): Store=92f3e9baa717d57147319661f6f00e06/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:52:36,330 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 92f3e9baa717d57147319661f6f00e06 2024-11-12T14:52:36,332 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/92f3e9baa717d57147319661f6f00e06 2024-11-12T14:52:36,336 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/92f3e9baa717d57147319661f6f00e06 2024-11-12T14:52:36,337 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/92f3e9baa717d57147319661f6f00e06/recovered.edits/0000000000000000004 2024-11-12T14:52:36,338 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e41036448a3677ed52f58f536690bf38/.tmp/colfamily11/a60e509a9bd64d8b95959288938738f2 is 52, key is zzz/colfamily11:colfamily11/1731422971748/Put/seqid=0 2024-11-12T14:52:36,345 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/92f3e9baa717d57147319661f6f00e06/recovered.edits/0000000000000000004 2024-11-12T14:52:36,345 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(2902): Flushing 92f3e9baa717d57147319661f6f00e06 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-12T14:52:36,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741880_1056 (size=4857) 2024-11-12T14:52:36,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741880_1056 (size=4857) 2024-11-12T14:52:36,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741880_1056 (size=4857) 2024-11-12T14:52:36,362 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e41036448a3677ed52f58f536690bf38/.tmp/colfamily11/a60e509a9bd64d8b95959288938738f2 2024-11-12T14:52:36,370 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/92f3e9baa717d57147319661f6f00e06/.tmp/colfamily11/4923b4eef211457e9178d8e604b0f297 is 52, key is jjj/colfamily11:colfamily11/1731422971566/Put/seqid=0 2024-11-12T14:52:36,373 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e41036448a3677ed52f58f536690bf38/.tmp/colfamily11/a60e509a9bd64d8b95959288938738f2 as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e41036448a3677ed52f58f536690bf38/colfamily11/a60e509a9bd64d8b95959288938738f2 2024-11-12T14:52:36,384 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e41036448a3677ed52f58f536690bf38/colfamily11/a60e509a9bd64d8b95959288938738f2, entries=1, sequenceid=4, filesize=4.7 K 2024-11-12T14:52:36,385 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~48 B/48, heapSize ~360 B/360, currentSize=0 B/0 for e41036448a3677ed52f58f536690bf38 in 69ms, sequenceid=4, compaction requested=false; wal=null 2024-11-12T14:52:36,386 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e41036448a3677ed52f58f536690bf38/recovered.edits/0000000000000000004 2024-11-12T14:52:36,388 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for e41036448a3677ed52f58f536690bf38 2024-11-12T14:52:36,388 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for e41036448a3677ed52f58f536690bf38 2024-11-12T14:52:36,389 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:52:36,390 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for e41036448a3677ed52f58f536690bf38 2024-11-12T14:52:36,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741881_1057 (size=39263) 2024-11-12T14:52:36,412 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e41036448a3677ed52f58f536690bf38/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T14:52:36,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741881_1057 (size=39263) 2024-11-12T14:52:36,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741881_1057 (size=39263) 2024-11-12T14:52:36,415 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened e41036448a3677ed52f58f536690bf38; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73161752, jitterRate=0.09019505977630615}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:52:36,415 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e41036448a3677ed52f58f536690bf38 2024-11-12T14:52:36,415 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for e41036448a3677ed52f58f536690bf38: Running coprocessor pre-open hook at 1731423156274Writing region info on filesystem at 1731423156274Initializing all the Stores at 1731423156275 (+1 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423156275Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423156285 (+10 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423156285Obtaining lock to block concurrent updates at 1731423156316 (+31 ms)Preparing flush snapshotting stores in e41036448a3677ed52f58f536690bf38 at 1731423156316Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38., syncing WAL and waiting on mvcc, flushsize=dataSize=48, getHeapSize=840, getOffHeapSize=0, getCellsCount=1 at 1731423156316Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38. at 1731423156316Flushing e41036448a3677ed52f58f536690bf38/colfamily11: creating writer at 1731423156316Flushing e41036448a3677ed52f58f536690bf38/colfamily11: appending metadata at 1731423156338 (+22 ms)Flushing e41036448a3677ed52f58f536690bf38/colfamily11: closing flushed file at 1731423156338Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32868e21: reopening flushed file at 1731423156371 (+33 ms)Finished flush of dataSize ~48 B/48, heapSize ~360 B/360, currentSize=0 B/0 for e41036448a3677ed52f58f536690bf38 in 69ms, sequenceid=4, compaction requested=false; wal=null at 1731423156385 (+14 ms)Cleaning up temporary data from old regions at 1731423156388 (+3 ms)Running coprocessor post-open hooks at 1731423156415 (+27 ms)Region opened successfully at 1731423156415 2024-11-12T14:52:36,415 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/92f3e9baa717d57147319661f6f00e06/.tmp/colfamily11/4923b4eef211457e9178d8e604b0f297 2024-11-12T14:52:36,416 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38., pid=73, masterSystemTime=1731423156124 2024-11-12T14:52:36,419 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38. 2024-11-12T14:52:36,419 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38. 2024-11-12T14:52:36,420 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33. 2024-11-12T14:52:36,420 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 06aa3b2e56ae4404b602d6fa98eabe33, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-12T14:52:36,420 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 06aa3b2e56ae4404b602d6fa98eabe33 2024-11-12T14:52:36,420 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:52:36,421 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 06aa3b2e56ae4404b602d6fa98eabe33 2024-11-12T14:52:36,421 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 06aa3b2e56ae4404b602d6fa98eabe33 2024-11-12T14:52:36,423 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=e41036448a3677ed52f58f536690bf38, regionState=OPEN, openSeqNum=5, regionLocation=3945e1ca6f47,45845,1731422964694 2024-11-12T14:52:36,427 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure e41036448a3677ed52f58f536690bf38, server=3945e1ca6f47,45845,1731422964694 because future has completed 2024-11-12T14:52:36,429 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38303 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=3945e1ca6f47,45845,1731422964694, table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e41036448a3677ed52f58f536690bf38. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T14:52:36,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=64 2024-11-12T14:52:36,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure e41036448a3677ed52f58f536690bf38, server=3945e1ca6f47,45845,1731422964694 in 460 msec 2024-11-12T14:52:36,433 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/92f3e9baa717d57147319661f6f00e06/.tmp/colfamily11/4923b4eef211457e9178d8e604b0f297 as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/92f3e9baa717d57147319661f6f00e06/colfamily11/4923b4eef211457e9178d8e604b0f297 2024-11-12T14:52:36,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e41036448a3677ed52f58f536690bf38, ASSIGN in 624 msec 2024-11-12T14:52:36,437 INFO [StoreOpener-06aa3b2e56ae4404b602d6fa98eabe33-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 06aa3b2e56ae4404b602d6fa98eabe33 2024-11-12T14:52:36,439 INFO [StoreOpener-06aa3b2e56ae4404b602d6fa98eabe33-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06aa3b2e56ae4404b602d6fa98eabe33 columnFamilyName colfamily11 2024-11-12T14:52:36,439 DEBUG [StoreOpener-06aa3b2e56ae4404b602d6fa98eabe33-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:36,441 INFO [StoreOpener-06aa3b2e56ae4404b602d6fa98eabe33-1 {}] regionserver.HStore(327): Store=06aa3b2e56ae4404b602d6fa98eabe33/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:52:36,441 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/92f3e9baa717d57147319661f6f00e06/colfamily11/4923b4eef211457e9178d8e604b0f297, entries=703, sequenceid=4, filesize=38.3 K 2024-11-12T14:52:36,441 INFO [StoreOpener-06aa3b2e56ae4404b602d6fa98eabe33-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 06aa3b2e56ae4404b602d6fa98eabe33 2024-11-12T14:52:36,441 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 92f3e9baa717d57147319661f6f00e06 in 96ms, sequenceid=4, compaction requested=false; wal=null 2024-11-12T14:52:36,442 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/92f3e9baa717d57147319661f6f00e06/recovered.edits/0000000000000000004 2024-11-12T14:52:36,444 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 92f3e9baa717d57147319661f6f00e06 2024-11-12T14:52:36,444 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 92f3e9baa717d57147319661f6f00e06 2024-11-12T14:52:36,445 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:52:36,448 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 92f3e9baa717d57147319661f6f00e06 2024-11-12T14:52:36,449 INFO [StoreOpener-06aa3b2e56ae4404b602d6fa98eabe33-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06aa3b2e56ae4404b602d6fa98eabe33 columnFamilyName colfamily21 2024-11-12T14:52:36,449 DEBUG [StoreOpener-06aa3b2e56ae4404b602d6fa98eabe33-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:36,450 INFO [StoreOpener-06aa3b2e56ae4404b602d6fa98eabe33-1 {}] regionserver.HStore(327): Store=06aa3b2e56ae4404b602d6fa98eabe33/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:52:36,450 INFO [StoreOpener-06aa3b2e56ae4404b602d6fa98eabe33-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 06aa3b2e56ae4404b602d6fa98eabe33 2024-11-12T14:52:36,451 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/92f3e9baa717d57147319661f6f00e06/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T14:52:36,452 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 92f3e9baa717d57147319661f6f00e06; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64025097, jitterRate=-0.045951709151268005}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:52:36,452 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 92f3e9baa717d57147319661f6f00e06 2024-11-12T14:52:36,452 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 92f3e9baa717d57147319661f6f00e06: Running coprocessor pre-open hook at 1731423156298Writing region info on filesystem at 1731423156298Initializing all the Stores at 1731423156300 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423156300Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423156301 (+1 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423156301Obtaining lock to block concurrent updates at 1731423156345 (+44 ms)Preparing flush snapshotting stores in 92f3e9baa717d57147319661f6f00e06 at 1731423156345Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731423156345Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06. at 1731423156345Flushing 92f3e9baa717d57147319661f6f00e06/colfamily11: creating writer at 1731423156345Flushing 92f3e9baa717d57147319661f6f00e06/colfamily11: appending metadata at 1731423156370 (+25 ms)Flushing 92f3e9baa717d57147319661f6f00e06/colfamily11: closing flushed file at 1731423156370Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66b9528: reopening flushed file at 1731423156431 (+61 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 92f3e9baa717d57147319661f6f00e06 in 96ms, sequenceid=4, compaction requested=false; wal=null at 1731423156442 (+11 ms)Cleaning up temporary data from old regions at 1731423156444 (+2 ms)Running coprocessor post-open hooks at 1731423156452 (+8 ms)Region opened successfully at 1731423156452 2024-11-12T14:52:36,455 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06., pid=75, masterSystemTime=1731423156141 2024-11-12T14:52:36,458 INFO [StoreOpener-06aa3b2e56ae4404b602d6fa98eabe33-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06aa3b2e56ae4404b602d6fa98eabe33 columnFamilyName colfamily31 2024-11-12T14:52:36,458 DEBUG [StoreOpener-06aa3b2e56ae4404b602d6fa98eabe33-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:36,458 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06. 2024-11-12T14:52:36,458 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06. 2024-11-12T14:52:36,459 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b. 2024-11-12T14:52:36,459 INFO [StoreOpener-06aa3b2e56ae4404b602d6fa98eabe33-1 {}] regionserver.HStore(327): Store=06aa3b2e56ae4404b602d6fa98eabe33/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:52:36,459 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 06aa3b2e56ae4404b602d6fa98eabe33 2024-11-12T14:52:36,459 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=92f3e9baa717d57147319661f6f00e06, regionState=OPEN, openSeqNum=5, regionLocation=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:36,460 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 9a6f7a226796c8ff7e21a40d463cd27b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-12T14:52:36,460 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06aa3b2e56ae4404b602d6fa98eabe33 2024-11-12T14:52:36,461 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 9a6f7a226796c8ff7e21a40d463cd27b 2024-11-12T14:52:36,461 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:52:36,461 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 9a6f7a226796c8ff7e21a40d463cd27b 2024-11-12T14:52:36,461 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 9a6f7a226796c8ff7e21a40d463cd27b 2024-11-12T14:52:36,461 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=75, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 92f3e9baa717d57147319661f6f00e06, server=3945e1ca6f47,35473,1731422964757 because future has completed 2024-11-12T14:52:36,463 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06aa3b2e56ae4404b602d6fa98eabe33 2024-11-12T14:52:36,464 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06aa3b2e56ae4404b602d6fa98eabe33/recovered.edits/0000000000000000004 2024-11-12T14:52:36,464 INFO [StoreOpener-9a6f7a226796c8ff7e21a40d463cd27b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 9a6f7a226796c8ff7e21a40d463cd27b 2024-11-12T14:52:36,466 INFO [StoreOpener-9a6f7a226796c8ff7e21a40d463cd27b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a6f7a226796c8ff7e21a40d463cd27b columnFamilyName colfamily11 2024-11-12T14:52:36,466 DEBUG [StoreOpener-9a6f7a226796c8ff7e21a40d463cd27b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:36,467 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=68 2024-11-12T14:52:36,467 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 92f3e9baa717d57147319661f6f00e06, server=3945e1ca6f47,35473,1731422964757 in 474 msec 2024-11-12T14:52:36,468 INFO [StoreOpener-9a6f7a226796c8ff7e21a40d463cd27b-1 {}] regionserver.HStore(327): Store=9a6f7a226796c8ff7e21a40d463cd27b/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:52:36,468 INFO [StoreOpener-9a6f7a226796c8ff7e21a40d463cd27b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 9a6f7a226796c8ff7e21a40d463cd27b 2024-11-12T14:52:36,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=92f3e9baa717d57147319661f6f00e06, ASSIGN in 659 msec 2024-11-12T14:52:36,473 INFO [StoreOpener-9a6f7a226796c8ff7e21a40d463cd27b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a6f7a226796c8ff7e21a40d463cd27b columnFamilyName colfamily21 2024-11-12T14:52:36,474 DEBUG [StoreOpener-9a6f7a226796c8ff7e21a40d463cd27b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:36,474 INFO [StoreOpener-9a6f7a226796c8ff7e21a40d463cd27b-1 {}] regionserver.HStore(327): Store=9a6f7a226796c8ff7e21a40d463cd27b/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:52:36,475 INFO [StoreOpener-9a6f7a226796c8ff7e21a40d463cd27b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 9a6f7a226796c8ff7e21a40d463cd27b 2024-11-12T14:52:36,475 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06aa3b2e56ae4404b602d6fa98eabe33/recovered.edits/0000000000000000004 2024-11-12T14:52:36,475 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(2902): Flushing 06aa3b2e56ae4404b602d6fa98eabe33 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-12T14:52:36,475 INFO [StoreOpener-9a6f7a226796c8ff7e21a40d463cd27b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a6f7a226796c8ff7e21a40d463cd27b columnFamilyName colfamily31 2024-11-12T14:52:36,475 DEBUG [StoreOpener-9a6f7a226796c8ff7e21a40d463cd27b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:36,476 INFO [StoreOpener-9a6f7a226796c8ff7e21a40d463cd27b-1 {}] regionserver.HStore(327): Store=9a6f7a226796c8ff7e21a40d463cd27b/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:52:36,476 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 9a6f7a226796c8ff7e21a40d463cd27b 2024-11-12T14:52:36,477 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9a6f7a226796c8ff7e21a40d463cd27b 2024-11-12T14:52:36,479 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9a6f7a226796c8ff7e21a40d463cd27b 2024-11-12T14:52:36,480 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9a6f7a226796c8ff7e21a40d463cd27b/recovered.edits/0000000000000000004 2024-11-12T14:52:36,487 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9a6f7a226796c8ff7e21a40d463cd27b/recovered.edits/0000000000000000004 2024-11-12T14:52:36,487 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(2902): Flushing 9a6f7a226796c8ff7e21a40d463cd27b 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-12T14:52:36,499 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06aa3b2e56ae4404b602d6fa98eabe33/.tmp/colfamily11/342a05ea50e345b6b01eb78f408e4c49 is 52, key is ccc/colfamily11:colfamily11/1731422971334/Put/seqid=0 2024-11-12T14:52:36,511 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9a6f7a226796c8ff7e21a40d463cd27b/.tmp/colfamily11/e143df31e6b94ed88bc7954dd17ecf8e is 52, key is www/colfamily11:colfamily11/1731422971740/Put/seqid=0 2024-11-12T14:52:36,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741882_1058 (size=39263) 2024-11-12T14:52:36,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741882_1058 (size=39263) 2024-11-12T14:52:36,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741882_1058 (size=39263) 2024-11-12T14:52:36,517 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06aa3b2e56ae4404b602d6fa98eabe33/.tmp/colfamily11/342a05ea50e345b6b01eb78f408e4c49 2024-11-12T14:52:36,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741883_1059 (size=39263) 2024-11-12T14:52:36,527 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06aa3b2e56ae4404b602d6fa98eabe33/.tmp/colfamily11/342a05ea50e345b6b01eb78f408e4c49 as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06aa3b2e56ae4404b602d6fa98eabe33/colfamily11/342a05ea50e345b6b01eb78f408e4c49 2024-11-12T14:52:36,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741883_1059 (size=39263) 2024-11-12T14:52:36,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741883_1059 (size=39263) 2024-11-12T14:52:36,529 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9a6f7a226796c8ff7e21a40d463cd27b/.tmp/colfamily11/e143df31e6b94ed88bc7954dd17ecf8e 2024-11-12T14:52:36,540 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9a6f7a226796c8ff7e21a40d463cd27b/.tmp/colfamily11/e143df31e6b94ed88bc7954dd17ecf8e as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9a6f7a226796c8ff7e21a40d463cd27b/colfamily11/e143df31e6b94ed88bc7954dd17ecf8e 2024-11-12T14:52:36,541 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06aa3b2e56ae4404b602d6fa98eabe33/colfamily11/342a05ea50e345b6b01eb78f408e4c49, entries=703, sequenceid=4, filesize=38.3 K 2024-11-12T14:52:36,542 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 06aa3b2e56ae4404b602d6fa98eabe33 in 66ms, sequenceid=4, compaction requested=false; wal=null 2024-11-12T14:52:36,542 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06aa3b2e56ae4404b602d6fa98eabe33/recovered.edits/0000000000000000004 2024-11-12T14:52:36,543 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 06aa3b2e56ae4404b602d6fa98eabe33 2024-11-12T14:52:36,544 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 06aa3b2e56ae4404b602d6fa98eabe33 2024-11-12T14:52:36,544 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:52:36,547 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 06aa3b2e56ae4404b602d6fa98eabe33 2024-11-12T14:52:36,549 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9a6f7a226796c8ff7e21a40d463cd27b/colfamily11/e143df31e6b94ed88bc7954dd17ecf8e, entries=703, sequenceid=4, filesize=38.3 K 2024-11-12T14:52:36,549 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 9a6f7a226796c8ff7e21a40d463cd27b in 62ms, sequenceid=4, compaction requested=false; wal=null 2024-11-12T14:52:36,550 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9a6f7a226796c8ff7e21a40d463cd27b/recovered.edits/0000000000000000004 2024-11-12T14:52:36,552 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 9a6f7a226796c8ff7e21a40d463cd27b 2024-11-12T14:52:36,552 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 9a6f7a226796c8ff7e21a40d463cd27b 2024-11-12T14:52:36,552 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:52:36,554 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06aa3b2e56ae4404b602d6fa98eabe33/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T14:52:36,554 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 9a6f7a226796c8ff7e21a40d463cd27b 2024-11-12T14:52:36,555 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 06aa3b2e56ae4404b602d6fa98eabe33; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66297262, jitterRate=-0.012093812227249146}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:52:36,555 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 06aa3b2e56ae4404b602d6fa98eabe33 2024-11-12T14:52:36,555 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 06aa3b2e56ae4404b602d6fa98eabe33: Running coprocessor pre-open hook at 1731423156421Writing region info on filesystem at 1731423156421Initializing all the Stores at 1731423156423 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423156423Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423156437 (+14 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423156437Obtaining lock to block concurrent updates at 1731423156475 (+38 ms)Preparing flush snapshotting stores in 06aa3b2e56ae4404b602d6fa98eabe33 at 1731423156475Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731423156475Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33. at 1731423156475Flushing 06aa3b2e56ae4404b602d6fa98eabe33/colfamily11: creating writer at 1731423156476 (+1 ms)Flushing 06aa3b2e56ae4404b602d6fa98eabe33/colfamily11: appending metadata at 1731423156499 (+23 ms)Flushing 06aa3b2e56ae4404b602d6fa98eabe33/colfamily11: closing flushed file at 1731423156499Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3de75a0d: reopening flushed file at 1731423156526 (+27 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 06aa3b2e56ae4404b602d6fa98eabe33 in 66ms, sequenceid=4, compaction requested=false; wal=null at 1731423156542 (+16 ms)Cleaning up temporary data from old regions at 1731423156544 (+2 ms)Running coprocessor post-open hooks at 1731423156555 (+11 ms)Region opened successfully at 1731423156555 2024-11-12T14:52:36,556 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33., pid=78, masterSystemTime=1731423156124 2024-11-12T14:52:36,558 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9a6f7a226796c8ff7e21a40d463cd27b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T14:52:36,559 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33. 2024-11-12T14:52:36,559 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33. 2024-11-12T14:52:36,560 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 9a6f7a226796c8ff7e21a40d463cd27b; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62771644, jitterRate=-0.06462961435317993}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:52:36,560 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b. 2024-11-12T14:52:36,560 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9a6f7a226796c8ff7e21a40d463cd27b 2024-11-12T14:52:36,560 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 9a6f7a226796c8ff7e21a40d463cd27b: Running coprocessor pre-open hook at 1731423156462Writing region info on filesystem at 1731423156462Initializing all the Stores at 1731423156464 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423156464Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423156464Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423156464Obtaining lock to block concurrent updates at 1731423156487 (+23 ms)Preparing flush snapshotting stores in 9a6f7a226796c8ff7e21a40d463cd27b at 1731423156487Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731423156488 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b. at 1731423156488Flushing 9a6f7a226796c8ff7e21a40d463cd27b/colfamily11: creating writer at 1731423156488Flushing 9a6f7a226796c8ff7e21a40d463cd27b/colfamily11: appending metadata at 1731423156511 (+23 ms)Flushing 9a6f7a226796c8ff7e21a40d463cd27b/colfamily11: closing flushed file at 1731423156511Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2658abc0: reopening flushed file at 1731423156539 (+28 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 9a6f7a226796c8ff7e21a40d463cd27b in 62ms, sequenceid=4, compaction requested=false; wal=null at 1731423156549 (+10 ms)Cleaning up temporary data from old regions at 1731423156552 (+3 ms)Running coprocessor post-open hooks at 1731423156560 (+8 ms)Region opened successfully at 1731423156560 2024-11-12T14:52:36,560 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => 2d931ca9d93687bd79e96c94cb16690b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-12T14:52:36,560 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=06aa3b2e56ae4404b602d6fa98eabe33, regionState=OPEN, openSeqNum=5, regionLocation=3945e1ca6f47,45845,1731422964694 2024-11-12T14:52:36,560 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2d931ca9d93687bd79e96c94cb16690b 2024-11-12T14:52:36,560 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:52:36,561 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for 2d931ca9d93687bd79e96c94cb16690b 2024-11-12T14:52:36,561 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for 2d931ca9d93687bd79e96c94cb16690b 2024-11-12T14:52:36,561 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b., pid=77, masterSystemTime=1731423156141 2024-11-12T14:52:36,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 06aa3b2e56ae4404b602d6fa98eabe33, server=3945e1ca6f47,45845,1731422964694 because future has completed 2024-11-12T14:52:36,564 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b. 2024-11-12T14:52:36,564 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b. 2024-11-12T14:52:36,564 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd. 2024-11-12T14:52:36,564 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=9a6f7a226796c8ff7e21a40d463cd27b, regionState=OPEN, openSeqNum=5, regionLocation=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:36,565 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 2cc2e73ccc39f945b27a1d50dd2694bd, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-12T14:52:36,565 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2cc2e73ccc39f945b27a1d50dd2694bd 2024-11-12T14:52:36,565 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:52:36,565 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 2cc2e73ccc39f945b27a1d50dd2694bd 2024-11-12T14:52:36,565 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 2cc2e73ccc39f945b27a1d50dd2694bd 2024-11-12T14:52:36,567 INFO [StoreOpener-2d931ca9d93687bd79e96c94cb16690b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 2d931ca9d93687bd79e96c94cb16690b 2024-11-12T14:52:36,567 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9a6f7a226796c8ff7e21a40d463cd27b, server=3945e1ca6f47,35473,1731422964757 because future has completed 2024-11-12T14:52:36,569 INFO [StoreOpener-2d931ca9d93687bd79e96c94cb16690b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d931ca9d93687bd79e96c94cb16690b columnFamilyName colfamily11 2024-11-12T14:52:36,569 DEBUG [StoreOpener-2d931ca9d93687bd79e96c94cb16690b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:36,570 INFO [StoreOpener-2d931ca9d93687bd79e96c94cb16690b-1 {}] regionserver.HStore(327): Store=2d931ca9d93687bd79e96c94cb16690b/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:52:36,570 INFO [StoreOpener-2d931ca9d93687bd79e96c94cb16690b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 2d931ca9d93687bd79e96c94cb16690b 2024-11-12T14:52:36,571 INFO [StoreOpener-2d931ca9d93687bd79e96c94cb16690b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d931ca9d93687bd79e96c94cb16690b columnFamilyName colfamily21 2024-11-12T14:52:36,572 DEBUG [StoreOpener-2d931ca9d93687bd79e96c94cb16690b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:36,572 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=70 2024-11-12T14:52:36,572 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure 06aa3b2e56ae4404b602d6fa98eabe33, server=3945e1ca6f47,45845,1731422964694 in 571 msec 2024-11-12T14:52:36,572 INFO [StoreOpener-2d931ca9d93687bd79e96c94cb16690b-1 {}] regionserver.HStore(327): Store=2d931ca9d93687bd79e96c94cb16690b/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:52:36,572 INFO [StoreOpener-2d931ca9d93687bd79e96c94cb16690b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 2d931ca9d93687bd79e96c94cb16690b 2024-11-12T14:52:36,573 INFO [StoreOpener-2cc2e73ccc39f945b27a1d50dd2694bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region 2cc2e73ccc39f945b27a1d50dd2694bd 2024-11-12T14:52:36,574 INFO [StoreOpener-2cc2e73ccc39f945b27a1d50dd2694bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2cc2e73ccc39f945b27a1d50dd2694bd columnFamilyName colfamily11 2024-11-12T14:52:36,574 DEBUG [StoreOpener-2cc2e73ccc39f945b27a1d50dd2694bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:36,574 INFO [StoreOpener-2d931ca9d93687bd79e96c94cb16690b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d931ca9d93687bd79e96c94cb16690b columnFamilyName colfamily31 2024-11-12T14:52:36,574 DEBUG [StoreOpener-2d931ca9d93687bd79e96c94cb16690b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:36,575 INFO [StoreOpener-2d931ca9d93687bd79e96c94cb16690b-1 {}] regionserver.HStore(327): Store=2d931ca9d93687bd79e96c94cb16690b/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:52:36,575 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for 2d931ca9d93687bd79e96c94cb16690b 2024-11-12T14:52:36,576 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2d931ca9d93687bd79e96c94cb16690b 2024-11-12T14:52:36,578 INFO [StoreOpener-2cc2e73ccc39f945b27a1d50dd2694bd-1 {}] regionserver.HStore(327): Store=2cc2e73ccc39f945b27a1d50dd2694bd/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:52:36,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=06aa3b2e56ae4404b602d6fa98eabe33, ASSIGN in 764 msec 2024-11-12T14:52:36,578 INFO [StoreOpener-2cc2e73ccc39f945b27a1d50dd2694bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region 2cc2e73ccc39f945b27a1d50dd2694bd 2024-11-12T14:52:36,579 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2d931ca9d93687bd79e96c94cb16690b 2024-11-12T14:52:36,580 INFO [StoreOpener-2cc2e73ccc39f945b27a1d50dd2694bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2cc2e73ccc39f945b27a1d50dd2694bd columnFamilyName colfamily21 2024-11-12T14:52:36,580 DEBUG [StoreOpener-2cc2e73ccc39f945b27a1d50dd2694bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:36,580 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2d931ca9d93687bd79e96c94cb16690b/recovered.edits/0000000000000000004 2024-11-12T14:52:36,580 INFO [StoreOpener-2cc2e73ccc39f945b27a1d50dd2694bd-1 {}] regionserver.HStore(327): Store=2cc2e73ccc39f945b27a1d50dd2694bd/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:52:36,581 INFO [StoreOpener-2cc2e73ccc39f945b27a1d50dd2694bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region 2cc2e73ccc39f945b27a1d50dd2694bd 2024-11-12T14:52:36,582 INFO [StoreOpener-2cc2e73ccc39f945b27a1d50dd2694bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2cc2e73ccc39f945b27a1d50dd2694bd columnFamilyName colfamily31 2024-11-12T14:52:36,582 DEBUG [StoreOpener-2cc2e73ccc39f945b27a1d50dd2694bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:36,584 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=66 2024-11-12T14:52:36,584 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure 9a6f7a226796c8ff7e21a40d463cd27b, server=3945e1ca6f47,35473,1731422964757 in 585 msec 2024-11-12T14:52:36,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9a6f7a226796c8ff7e21a40d463cd27b, ASSIGN in 776 msec 2024-11-12T14:52:36,593 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2d931ca9d93687bd79e96c94cb16690b/recovered.edits/0000000000000000004 2024-11-12T14:52:36,594 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(2902): Flushing 2d931ca9d93687bd79e96c94cb16690b 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-12T14:52:36,597 INFO [StoreOpener-2cc2e73ccc39f945b27a1d50dd2694bd-1 {}] regionserver.HStore(327): Store=2cc2e73ccc39f945b27a1d50dd2694bd/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:52:36,597 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 2cc2e73ccc39f945b27a1d50dd2694bd 2024-11-12T14:52:36,598 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cc2e73ccc39f945b27a1d50dd2694bd 2024-11-12T14:52:36,600 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cc2e73ccc39f945b27a1d50dd2694bd 2024-11-12T14:52:36,601 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cc2e73ccc39f945b27a1d50dd2694bd/recovered.edits/0000000000000000004 2024-11-12T14:52:36,609 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cc2e73ccc39f945b27a1d50dd2694bd/recovered.edits/0000000000000000004 2024-11-12T14:52:36,609 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(2902): Flushing 2cc2e73ccc39f945b27a1d50dd2694bd 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-12T14:52:36,617 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2d931ca9d93687bd79e96c94cb16690b/.tmp/colfamily11/ae28f4e7929245859ede3e486c88e90c is 52, key is ttt/colfamily11:colfamily11/1731422971694/Put/seqid=0 2024-11-12T14:52:36,635 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cc2e73ccc39f945b27a1d50dd2694bd/.tmp/colfamily11/02fb21b4d79349ab9f1a3a0520b822fe is 52, key is qqq/colfamily11:colfamily11/1731422971646/Put/seqid=0 2024-11-12T14:52:36,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741884_1060 (size=39263) 2024-11-12T14:52:36,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741884_1060 (size=39263) 2024-11-12T14:52:36,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741884_1060 (size=39263) 2024-11-12T14:52:36,663 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2d931ca9d93687bd79e96c94cb16690b/.tmp/colfamily11/ae28f4e7929245859ede3e486c88e90c 2024-11-12T14:52:36,672 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2d931ca9d93687bd79e96c94cb16690b/.tmp/colfamily11/ae28f4e7929245859ede3e486c88e90c as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2d931ca9d93687bd79e96c94cb16690b/colfamily11/ae28f4e7929245859ede3e486c88e90c 2024-11-12T14:52:36,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741885_1061 (size=39263) 2024-11-12T14:52:36,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741885_1061 (size=39263) 2024-11-12T14:52:36,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741885_1061 (size=39263) 2024-11-12T14:52:36,680 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cc2e73ccc39f945b27a1d50dd2694bd/.tmp/colfamily11/02fb21b4d79349ab9f1a3a0520b822fe 2024-11-12T14:52:36,685 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2d931ca9d93687bd79e96c94cb16690b/colfamily11/ae28f4e7929245859ede3e486c88e90c, entries=703, sequenceid=4, filesize=38.3 K 2024-11-12T14:52:36,686 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 2d931ca9d93687bd79e96c94cb16690b in 91ms, sequenceid=4, compaction requested=false; wal=null 2024-11-12T14:52:36,686 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2d931ca9d93687bd79e96c94cb16690b/recovered.edits/0000000000000000004 2024-11-12T14:52:36,688 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for 2d931ca9d93687bd79e96c94cb16690b 2024-11-12T14:52:36,688 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for 2d931ca9d93687bd79e96c94cb16690b 2024-11-12T14:52:36,689 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:52:36,691 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for 2d931ca9d93687bd79e96c94cb16690b 2024-11-12T14:52:36,692 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cc2e73ccc39f945b27a1d50dd2694bd/.tmp/colfamily11/02fb21b4d79349ab9f1a3a0520b822fe as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cc2e73ccc39f945b27a1d50dd2694bd/colfamily11/02fb21b4d79349ab9f1a3a0520b822fe 2024-11-12T14:52:36,699 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2d931ca9d93687bd79e96c94cb16690b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T14:52:36,701 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened 2d931ca9d93687bd79e96c94cb16690b; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72393695, jitterRate=0.0787501186132431}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:52:36,702 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2d931ca9d93687bd79e96c94cb16690b 2024-11-12T14:52:36,702 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for 2d931ca9d93687bd79e96c94cb16690b: Running coprocessor pre-open hook at 1731423156561Writing region info on filesystem at 1731423156561Initializing all the Stores at 1731423156563 (+2 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423156564 (+1 ms)Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423156566 (+2 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423156566Obtaining lock to block concurrent updates at 1731423156594 (+28 ms)Preparing flush snapshotting stores in 2d931ca9d93687bd79e96c94cb16690b at 1731423156595 (+1 ms)Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731423156595Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b. at 1731423156595Flushing 2d931ca9d93687bd79e96c94cb16690b/colfamily11: creating writer at 1731423156595Flushing 2d931ca9d93687bd79e96c94cb16690b/colfamily11: appending metadata at 1731423156617 (+22 ms)Flushing 2d931ca9d93687bd79e96c94cb16690b/colfamily11: closing flushed file at 1731423156617Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d077150: reopening flushed file at 1731423156670 (+53 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 2d931ca9d93687bd79e96c94cb16690b in 91ms, sequenceid=4, compaction requested=false; wal=null at 1731423156686 (+16 ms)Cleaning up temporary data from old regions at 1731423156688 (+2 ms)Running coprocessor post-open hooks at 1731423156702 (+14 ms)Region opened successfully at 1731423156702 2024-11-12T14:52:36,703 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b., pid=80, masterSystemTime=1731423156124 2024-11-12T14:52:36,703 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cc2e73ccc39f945b27a1d50dd2694bd/colfamily11/02fb21b4d79349ab9f1a3a0520b822fe, entries=703, sequenceid=4, filesize=38.3 K 2024-11-12T14:52:36,704 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 2cc2e73ccc39f945b27a1d50dd2694bd in 94ms, sequenceid=4, compaction requested=false; wal=null 2024-11-12T14:52:36,705 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cc2e73ccc39f945b27a1d50dd2694bd/recovered.edits/0000000000000000004 2024-11-12T14:52:36,706 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b. 2024-11-12T14:52:36,706 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b. 2024-11-12T14:52:36,706 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b. 2024-11-12T14:52:36,706 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => d7c3d3ae3b942f0414dcbb110be8bc0b, NAME => 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-12T14:52:36,707 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true d7c3d3ae3b942f0414dcbb110be8bc0b 2024-11-12T14:52:36,707 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:52:36,707 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for d7c3d3ae3b942f0414dcbb110be8bc0b 2024-11-12T14:52:36,707 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for d7c3d3ae3b942f0414dcbb110be8bc0b 2024-11-12T14:52:36,708 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=2d931ca9d93687bd79e96c94cb16690b, regionState=OPEN, openSeqNum=5, regionLocation=3945e1ca6f47,45845,1731422964694 2024-11-12T14:52:36,709 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 2cc2e73ccc39f945b27a1d50dd2694bd 2024-11-12T14:52:36,709 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 2cc2e73ccc39f945b27a1d50dd2694bd 2024-11-12T14:52:36,711 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:52:36,711 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d931ca9d93687bd79e96c94cb16690b, server=3945e1ca6f47,45845,1731422964694 because future has completed 2024-11-12T14:52:36,714 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 2cc2e73ccc39f945b27a1d50dd2694bd 2024-11-12T14:52:36,716 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=69 2024-11-12T14:52:36,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure 2d931ca9d93687bd79e96c94cb16690b, server=3945e1ca6f47,45845,1731422964694 in 711 msec 2024-11-12T14:52:36,717 INFO [StoreOpener-d7c3d3ae3b942f0414dcbb110be8bc0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily11 of region d7c3d3ae3b942f0414dcbb110be8bc0b 2024-11-12T14:52:36,720 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cc2e73ccc39f945b27a1d50dd2694bd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T14:52:36,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2d931ca9d93687bd79e96c94cb16690b, ASSIGN in 909 msec 2024-11-12T14:52:36,721 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 2cc2e73ccc39f945b27a1d50dd2694bd; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65492158, jitterRate=-0.02409079670906067}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:52:36,721 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2cc2e73ccc39f945b27a1d50dd2694bd 2024-11-12T14:52:36,722 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 2cc2e73ccc39f945b27a1d50dd2694bd: Running coprocessor pre-open hook at 1731423156565Writing region info on filesystem at 1731423156565Initializing all the Stores at 1731423156568 (+3 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423156568Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423156573 (+5 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423156573Obtaining lock to block concurrent updates at 1731423156610 (+37 ms)Preparing flush snapshotting stores in 2cc2e73ccc39f945b27a1d50dd2694bd at 1731423156610Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731423156610Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd. at 1731423156610Flushing 2cc2e73ccc39f945b27a1d50dd2694bd/colfamily11: creating writer at 1731423156610Flushing 2cc2e73ccc39f945b27a1d50dd2694bd/colfamily11: appending metadata at 1731423156635 (+25 ms)Flushing 2cc2e73ccc39f945b27a1d50dd2694bd/colfamily11: closing flushed file at 1731423156635Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5cad47e1: reopening flushed file at 1731423156691 (+56 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 2cc2e73ccc39f945b27a1d50dd2694bd in 94ms, sequenceid=4, compaction requested=false; wal=null at 1731423156704 (+13 ms)Cleaning up temporary data from old regions at 1731423156709 (+5 ms)Running coprocessor post-open hooks at 1731423156722 (+13 ms)Region opened successfully at 1731423156722 2024-11-12T14:52:36,723 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd., pid=76, masterSystemTime=1731423156141 2024-11-12T14:52:36,726 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd. 2024-11-12T14:52:36,726 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd. 2024-11-12T14:52:36,727 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=2cc2e73ccc39f945b27a1d50dd2694bd, regionState=OPEN, openSeqNum=5, regionLocation=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:36,727 INFO [StoreOpener-d7c3d3ae3b942f0414dcbb110be8bc0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7c3d3ae3b942f0414dcbb110be8bc0b columnFamilyName colfamily11 2024-11-12T14:52:36,727 DEBUG [StoreOpener-d7c3d3ae3b942f0414dcbb110be8bc0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:36,728 INFO [StoreOpener-d7c3d3ae3b942f0414dcbb110be8bc0b-1 {}] regionserver.HStore(327): Store=d7c3d3ae3b942f0414dcbb110be8bc0b/colfamily11, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:52:36,728 INFO [StoreOpener-d7c3d3ae3b942f0414dcbb110be8bc0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily21 of region d7c3d3ae3b942f0414dcbb110be8bc0b 2024-11-12T14:52:36,730 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=76, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2cc2e73ccc39f945b27a1d50dd2694bd, server=3945e1ca6f47,35473,1731422964757 because future has completed 2024-11-12T14:52:36,731 INFO [StoreOpener-d7c3d3ae3b942f0414dcbb110be8bc0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7c3d3ae3b942f0414dcbb110be8bc0b columnFamilyName colfamily21 2024-11-12T14:52:36,731 DEBUG [StoreOpener-d7c3d3ae3b942f0414dcbb110be8bc0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:36,732 INFO [StoreOpener-d7c3d3ae3b942f0414dcbb110be8bc0b-1 {}] regionserver.HStore(327): Store=d7c3d3ae3b942f0414dcbb110be8bc0b/colfamily21, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:52:36,732 INFO [StoreOpener-d7c3d3ae3b942f0414dcbb110be8bc0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family colfamily31 of region d7c3d3ae3b942f0414dcbb110be8bc0b 2024-11-12T14:52:36,733 INFO [StoreOpener-d7c3d3ae3b942f0414dcbb110be8bc0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7c3d3ae3b942f0414dcbb110be8bc0b columnFamilyName colfamily31 2024-11-12T14:52:36,734 DEBUG [StoreOpener-d7c3d3ae3b942f0414dcbb110be8bc0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:52:36,737 INFO [StoreOpener-d7c3d3ae3b942f0414dcbb110be8bc0b-1 {}] regionserver.HStore(327): Store=d7c3d3ae3b942f0414dcbb110be8bc0b/colfamily31, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:52:36,737 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for d7c3d3ae3b942f0414dcbb110be8bc0b 2024-11-12T14:52:36,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=65 2024-11-12T14:52:36,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=65, state=SUCCESS, hasLock=false; OpenRegionProcedure 2cc2e73ccc39f945b27a1d50dd2694bd, server=3945e1ca6f47,35473,1731422964757 in 741 msec 2024-11-12T14:52:36,738 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d7c3d3ae3b942f0414dcbb110be8bc0b 2024-11-12T14:52:36,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2cc2e73ccc39f945b27a1d50dd2694bd, ASSIGN in 930 msec 2024-11-12T14:52:36,740 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d7c3d3ae3b942f0414dcbb110be8bc0b 2024-11-12T14:52:36,741 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d7c3d3ae3b942f0414dcbb110be8bc0b/recovered.edits/0000000000000000004 2024-11-12T14:52:36,747 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5793): Applied 703, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=4, path=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d7c3d3ae3b942f0414dcbb110be8bc0b/recovered.edits/0000000000000000004 2024-11-12T14:52:36,748 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(2902): Flushing d7c3d3ae3b942f0414dcbb110be8bc0b 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-12T14:52:36,770 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d7c3d3ae3b942f0414dcbb110be8bc0b/.tmp/colfamily11/9713e207d53e485eb7f09081523ee4cb is 52, key is nnn/colfamily11:colfamily11/1731422971605/Put/seqid=0 2024-11-12T14:52:36,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741886_1062 (size=39263) 2024-11-12T14:52:36,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741886_1062 (size=39263) 2024-11-12T14:52:36,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741886_1062 (size=39263) 2024-11-12T14:52:36,790 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=4 (bloomFilter=false), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d7c3d3ae3b942f0414dcbb110be8bc0b/.tmp/colfamily11/9713e207d53e485eb7f09081523ee4cb 2024-11-12T14:52:36,799 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d7c3d3ae3b942f0414dcbb110be8bc0b/.tmp/colfamily11/9713e207d53e485eb7f09081523ee4cb as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d7c3d3ae3b942f0414dcbb110be8bc0b/colfamily11/9713e207d53e485eb7f09081523ee4cb 2024-11-12T14:52:36,810 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d7c3d3ae3b942f0414dcbb110be8bc0b/colfamily11/9713e207d53e485eb7f09081523ee4cb, entries=703, sequenceid=4, filesize=38.3 K 2024-11-12T14:52:36,811 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for d7c3d3ae3b942f0414dcbb110be8bc0b in 62ms, sequenceid=4, compaction requested=false; wal=null 2024-11-12T14:52:36,814 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d7c3d3ae3b942f0414dcbb110be8bc0b/recovered.edits/0000000000000000004 2024-11-12T14:52:36,817 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for d7c3d3ae3b942f0414dcbb110be8bc0b 2024-11-12T14:52:36,817 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for d7c3d3ae3b942f0414dcbb110be8bc0b 2024-11-12T14:52:36,818 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T14:52:36,821 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for d7c3d3ae3b942f0414dcbb110be8bc0b 2024-11-12T14:52:36,826 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d7c3d3ae3b942f0414dcbb110be8bc0b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T14:52:36,827 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened d7c3d3ae3b942f0414dcbb110be8bc0b; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74523110, jitterRate=0.11048087477684021}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T14:52:36,827 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d7c3d3ae3b942f0414dcbb110be8bc0b 2024-11-12T14:52:36,827 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for d7c3d3ae3b942f0414dcbb110be8bc0b: Running coprocessor pre-open hook at 1731423156707Writing region info on filesystem at 1731423156707Initializing all the Stores at 1731423156711 (+4 ms)Instantiating store for column family {NAME => 'colfamily11', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423156711Instantiating store for column family {NAME => 'colfamily21', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423156717 (+6 ms)Instantiating store for column family {NAME => 'colfamily31', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731423156717Obtaining lock to block concurrent updates at 1731423156748 (+31 ms)Preparing flush snapshotting stores in d7c3d3ae3b942f0414dcbb110be8bc0b at 1731423156748Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731423156748Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b. at 1731423156748Flushing d7c3d3ae3b942f0414dcbb110be8bc0b/colfamily11: creating writer at 1731423156748Flushing d7c3d3ae3b942f0414dcbb110be8bc0b/colfamily11: appending metadata at 1731423156770 (+22 ms)Flushing d7c3d3ae3b942f0414dcbb110be8bc0b/colfamily11: closing flushed file at 1731423156770Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a0b4f6c: reopening flushed file at 1731423156798 (+28 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for d7c3d3ae3b942f0414dcbb110be8bc0b in 62ms, sequenceid=4, compaction requested=false; wal=null at 1731423156811 (+13 ms)Cleaning up temporary data from old regions at 1731423156817 (+6 ms)Running coprocessor post-open hooks at 1731423156827 (+10 ms)Region opened successfully at 1731423156827 2024-11-12T14:52:36,828 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b., pid=79, masterSystemTime=1731423156124 2024-11-12T14:52:36,831 DEBUG [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b. 2024-11-12T14:52:36,832 INFO [RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b. 2024-11-12T14:52:36,832 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=d7c3d3ae3b942f0414dcbb110be8bc0b, regionState=OPEN, openSeqNum=5, regionLocation=3945e1ca6f47,45845,1731422964694 2024-11-12T14:52:36,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure d7c3d3ae3b942f0414dcbb110be8bc0b, server=3945e1ca6f47,45845,1731422964694 because future has completed 2024-11-12T14:52:36,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=72 2024-11-12T14:52:36,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=72, state=SUCCESS, hasLock=false; OpenRegionProcedure d7c3d3ae3b942f0414dcbb110be8bc0b, server=3945e1ca6f47,45845,1731422964694 in 836 msec 2024-11-12T14:52:36,843 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=72, resume processing ppid=57 2024-11-12T14:52:36,843 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d7c3d3ae3b942f0414dcbb110be8bc0b, ASSIGN in 1.0320 sec 2024-11-12T14:52:36,843 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 3945e1ca6f47,37185,1731422964568 after splitting done 2024-11-12T14:52:36,843 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: false 2024-11-12T14:52:36,843 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update: pid=57, state=SUCCESS, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true 2024-11-12T14:52:36,843 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T14:52:36,844 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update: pid=57, state=SUCCESS, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1879) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] 2024-11-12T14:52:37,058 INFO [Time-limited test {}] procedure.MasterProcedureTestingUtility(473): Restart 8 exec state=pid=57, state=SUCCESS, hasLock=true; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true 2024-11-12T14:52:37,058 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-12T14:52:37,058 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T14:52:37,059 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T14:52:37,059 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T14:52:37,059 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-12T14:52:37,059 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-12T14:52:37,059 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-12T14:52:37,060 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T14:52:37,061 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-12T14:52:37,065 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-12T14:52:37,065 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T14:52:37,066 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T14:52:37,066 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T14:52:37,066 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T14:52:37,066 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T14:52:37,066 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T14:52:37,066 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T14:52:37,066 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T14:52:37,066 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T14:52:37,066 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T14:52:37,067 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T14:52:37,067 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T14:52:37,067 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T14:52:37,067 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T14:52:37,067 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T14:52:37,067 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T14:52:37,067 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T14:52:37,067 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T14:52:37,067 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T14:52:37,067 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T14:52:37,067 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-12T14:52:37,067 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-12T14:52:37,067 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-12T14:52:37,068 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-12T14:52:37,068 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T14:52:37,069 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true 2024-11-12T14:52:37,071 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=RUNNABLE:SERVER_CRASH_FINISH, hasLock=false; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true 2024-11-12T14:52:37,071 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:37,071 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:37,071 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T14:52:37,071 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3945e1ca6f47,45845,1731422964694 2024-11-12T14:52:37,071 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3945e1ca6f47%2C37185%2C1731422964568.1731422966319, worker=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:37,071 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3945e1ca6f47%2C37185%2C1731422964568.1731422966319, worker=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e41036448a3677ed52f58f536690bf38, ASSIGN 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2cc2e73ccc39f945b27a1d50dd2694bd, ASSIGN 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9a6f7a226796c8ff7e21a40d463cd27b, ASSIGN 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4936dfde0b7af7ea99d36c9e0c24fefc, ASSIGN 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=92f3e9baa717d57147319661f6f00e06, ASSIGN 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2d931ca9d93687bd79e96c94cb16690b, ASSIGN 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=06aa3b2e56ae4404b602d6fa98eabe33, ASSIGN 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d3c298441d993e84c2d9e579914483db, ASSIGN 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d7c3d3ae3b942f0414dcbb110be8bc0b, ASSIGN 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure e41036448a3677ed52f58f536690bf38, server=3945e1ca6f47,45845,1731422964694 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure 4936dfde0b7af7ea99d36c9e0c24fefc, server=3945e1ca6f47,45845,1731422964694 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 92f3e9baa717d57147319661f6f00e06, server=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=65, state=SUCCESS, hasLock=false; OpenRegionProcedure 2cc2e73ccc39f945b27a1d50dd2694bd, server=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure 9a6f7a226796c8ff7e21a40d463cd27b, server=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure 06aa3b2e56ae4404b602d6fa98eabe33, server=3945e1ca6f47,45845,1731422964694 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=72, state=SUCCESS, hasLock=false; OpenRegionProcedure d7c3d3ae3b942f0414dcbb110be8bc0b, server=3945e1ca6f47,45845,1731422964694 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure 2d931ca9d93687bd79e96c94cb16690b, server=3945e1ca6f47,45845,1731422964694 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=71, state=SUCCESS, hasLock=false; OpenRegionProcedure d3c298441d993e84c2d9e579914483db, server=3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=RUNNABLE:SERVER_CRASH_FINISH, hasLock=false; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3945e1ca6f47%2C37185%2C1731422964568.meta.1731422966737.meta, worker=3945e1ca6f47,35473,1731422964757 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=3945e1ca6f47,45845,1731422964694 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 3945e1ca6f47%2C37185%2C1731422964568.1731422966319, worker=3945e1ca6f47,35473,1731422964757 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 3945e1ca6f47%2C37185%2C1731422964568.1731422966319, worker=3945e1ca6f47,35473,1731422964757 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=e41036448a3677ed52f58f536690bf38, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2cc2e73ccc39f945b27a1d50dd2694bd, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=9a6f7a226796c8ff7e21a40d463cd27b, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=67, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=4936dfde0b7af7ea99d36c9e0c24fefc, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=68, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=92f3e9baa717d57147319661f6f00e06, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=69, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=2d931ca9d93687bd79e96c94cb16690b, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=70, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=06aa3b2e56ae4404b602d6fa98eabe33, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=71, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d3c298441d993e84c2d9e579914483db, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true, region=d7c3d3ae3b942f0414dcbb110be8bc0b, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure e41036448a3677ed52f58f536690bf38, server=3945e1ca6f47,45845,1731422964694 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=SUCCESS, hasLock=false; OpenRegionProcedure 4936dfde0b7af7ea99d36c9e0c24fefc, server=3945e1ca6f47,45845,1731422964694 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 92f3e9baa717d57147319661f6f00e06, server=3945e1ca6f47,35473,1731422964757 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=65, state=SUCCESS, hasLock=false; OpenRegionProcedure 2cc2e73ccc39f945b27a1d50dd2694bd, server=3945e1ca6f47,35473,1731422964757 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:37,072 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=66, state=SUCCESS, hasLock=false; OpenRegionProcedure 9a6f7a226796c8ff7e21a40d463cd27b, server=3945e1ca6f47,35473,1731422964757 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:37,073 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=70, state=SUCCESS, hasLock=false; OpenRegionProcedure 06aa3b2e56ae4404b602d6fa98eabe33, server=3945e1ca6f47,45845,1731422964694 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:37,073 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=72, state=SUCCESS, hasLock=false; OpenRegionProcedure d7c3d3ae3b942f0414dcbb110be8bc0b, server=3945e1ca6f47,45845,1731422964694 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:37,073 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=69, state=SUCCESS, hasLock=false; OpenRegionProcedure 2d931ca9d93687bd79e96c94cb16690b, server=3945e1ca6f47,45845,1731422964694 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:37,073 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=71, state=SUCCESS, hasLock=false; OpenRegionProcedure d3c298441d993e84c2d9e579914483db, server=3945e1ca6f47,35473,1731422964757 didn't hold the lock before restarting, skip acquiring lock. 2024-11-12T14:52:37,073 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-12T14:52:37,073 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=61 2024-11-12T14:52:37,073 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=3945e1ca6f47,45845,1731422964694, table=hbase:meta, region=1588230740 2024-11-12T14:52:37,073 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 3945e1ca6f47,45845,1731422964694, state=OPEN 2024-11-12T14:52:37,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:52:37,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:52:37,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:52:37,075 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:52:37,075 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:52:37,075 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:52:37,075 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731423187075 2024-11-12T14:52:37,075 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-12T14:52:37,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f943ce043844f379c1b7ecbf76b5a419, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:37,085 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=489408be5f35a96bee1ee0f68f064599, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:37,086 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=06aa3b2e56ae4404b602d6fa98eabe33, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=5 2024-11-12T14:52:37,086 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cf0006d9b8777107f458b7008fc42ba9, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:37,086 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d3c298441d993e84c2d9e579914483db, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=5 2024-11-12T14:52:37,086 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=59fa7126e2ade1f9daaaeb8bd38b09e4, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:37,086 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4936dfde0b7af7ea99d36c9e0c24fefc, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=5 2024-11-12T14:52:37,086 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f65e71cd750425cce9713e30a75c6002, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:37,087 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=df409d6fa4df5d8fd91534d2d90eda60, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:37,087 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=92f3e9baa717d57147319661f6f00e06, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=5 2024-11-12T14:52:37,087 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eaeef83a325d6b259b1b899f552ba8d0, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:37,087 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51b19d6922dec6f0774f3d761593de8f, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:37,087 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ac225c32dbf496a52e546c7e0b1a05af, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:37,087 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d7c3d3ae3b942f0414dcbb110be8bc0b, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=5 2024-11-12T14:52:37,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb0382e4edc08a7c43f0571deb2374a4, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:37,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b419710b4e7b088b11da63dff9b7760e, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:37,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2cc2e73ccc39f945b27a1d50dd2694bd, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=5 2024-11-12T14:52:37,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a2cc208a6afe25c72a2045fb03e0ecb1, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:37,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b5e68238ca126132dbf55f0fbf38047, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:37,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2d931ca9d93687bd79e96c94cb16690b, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=5 2024-11-12T14:52:37,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=efc23c8cc5ebe3a5c96ef5d199223832, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:37,088 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2b04a6d83bd2998d75a1743bdf61651e, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:37,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9a6f7a226796c8ff7e21a40d463cd27b, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=5 2024-11-12T14:52:37,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=aa9c293d6596b8e2756cafba09b3d7dd, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=2 2024-11-12T14:52:37,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=95e5de757651e0aa099998abaa3b598f, regionState=OPEN, lastHost=3945e1ca6f47,35473,1731422964757, regionLocation=3945e1ca6f47,35473,1731422964757, openSeqNum=2 2024-11-12T14:52:37,089 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e41036448a3677ed52f58f536690bf38, regionState=OPEN, lastHost=3945e1ca6f47,45845,1731422964694, regionLocation=3945e1ca6f47,45845,1731422964694, openSeqNum=5 2024-11-12T14:52:37,089 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-12T14:52:37,089 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731423217089 2024-11-12T14:52:37,089 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731423277089 2024-11-12T14:52:37,089 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 13 msec 2024-11-12T14:52:37,089 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 3945e1ca6f47,37185,1731422964568 after splitting done 2024-11-12T14:52:37,089 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(132): Toggle KILL before store update to: true 2024-11-12T14:52:37,093 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; ServerCrashProcedure 3945e1ca6f47,37185,1731422964568, splitWal=true, meta=true in 3 mins, 4.53 sec 2024-11-12T14:52:37,345 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40668, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T14:52:37,349 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:52:37,356 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:52:37,362 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 3945e1ca6f47:37185 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 3945e1ca6f47/172.17.0.2:37185 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:52:37,363 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2, error=java.net.ConnectException: Call to address=3945e1ca6f47:37185 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:52:37,363 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2 is java.net.ConnectException: Connection refused 2024-11-12T14:52:37,363 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2 from cache 2024-11-12T14:52:37,363 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address 3945e1ca6f47:37185 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 3945e1ca6f47/172.17.0.2:37185 2024-11-12T14:52:37,469 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3945e1ca6f47:37185 this server is in the failed servers list 2024-11-12T14:52:37,469 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:52:37,469 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:52:37,469 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,37185,1731422964568, seqNum=-1 from cache 2024-11-12T14:52:37,577 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:52:37,580 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40684, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:52:37,581 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=3945e1ca6f47,45845,1731422964694, seqNum=-1] 2024-11-12T14:52:37,587 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ccc', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33., hostname=3945e1ca6f47,45845,1731422964694, seqNum=5] 2024-11-12T14:52:37,605 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3945e1ca6f47:37185 this server is in the failed servers list 2024-11-12T14:52:37,605 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:52:37,605 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:52:37,605 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2 from cache 2024-11-12T14:52:37,715 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='eee', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db., hostname=3945e1ca6f47,35473,1731422964757, seqNum=5] 2024-11-12T14:52:37,726 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3945e1ca6f47:37185 this server is in the failed servers list 2024-11-12T14:52:37,726 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:52:37,726 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:52:37,726 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2 from cache 2024-11-12T14:52:37,832 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ggg', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc., hostname=3945e1ca6f47,45845,1731422964694, seqNum=5] 2024-11-12T14:52:37,844 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3945e1ca6f47:37185 this server is in the failed servers list 2024-11-12T14:52:37,844 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:52:37,845 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:52:37,845 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2 from cache 2024-11-12T14:52:37,952 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='jjj', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06., hostname=3945e1ca6f47,35473,1731422964757, seqNum=5] 2024-11-12T14:52:37,980 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3945e1ca6f47:37185 this server is in the failed servers list 2024-11-12T14:52:37,981 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:52:37,981 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:52:37,981 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2 from cache 2024-11-12T14:52:38,092 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='nnn', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b., hostname=3945e1ca6f47,45845,1731422964694, seqNum=5] 2024-11-12T14:52:38,116 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3945e1ca6f47:37185 this server is in the failed servers list 2024-11-12T14:52:38,117 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:52:38,117 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:52:38,117 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2 from cache 2024-11-12T14:52:38,232 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='qqq', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd., hostname=3945e1ca6f47,35473,1731422964757, seqNum=5] 2024-11-12T14:52:38,253 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3945e1ca6f47:37185 this server is in the failed servers list 2024-11-12T14:52:38,253 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:52:38,253 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:52:38,254 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2 from cache 2024-11-12T14:52:38,364 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='ttt', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b., hostname=3945e1ca6f47,45845,1731422964694, seqNum=5] 2024-11-12T14:52:38,376 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3945e1ca6f47:37185 this server is in the failed servers list 2024-11-12T14:52:38,376 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:52:38,377 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:52:38,377 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2 from cache 2024-11-12T14:52:38,497 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='www', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b., hostname=3945e1ca6f47,35473,1731422964757, seqNum=5] 2024-11-12T14:52:38,509 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.AbstractRpcClient(357): Not trying to connect to 3945e1ca6f47:37185 this server is in the failed servers list 2024-11-12T14:52:38,509 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2 , the old value is region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=3945e1ca6f47:37185 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:52:38,509 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 3945e1ca6f47:37185 2024-11-12T14:52:38,509 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38., hostname=3945e1ca6f47,37185,1731422964568, seqNum=2 from cache 2024-11-12T14:52:38,628 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true', row='zzz', locateType=CURRENT is [region=testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38., hostname=3945e1ca6f47,45845,1731422964694, seqNum=5] 2024-11-12T14:52:38,668 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T14:52:38,719 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-12T14:52:38,719 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-12T14:52:38,719 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T14:52:38,719 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.procedure.TestSCPBase.tearDown(TestSCPBase.java:72) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T14:52:38,719 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:52:38,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:52:38,720 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:52:38,720 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-12T14:52:38,720 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1871265054, stopped=false 2024-11-12T14:52:38,720 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=3945e1ca6f47,38303,1731422963767 2024-11-12T14:52:38,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T14:52:38,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T14:52:38,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:52:38,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:52:38,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T14:52:38,722 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T14:52:38,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:52:38,722 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T14:52:38,722 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.procedure.TestSCPBase.tearDown(TestSCPBase.java:72) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T14:52:38,722 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:52:38,722 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3945e1ca6f47,45845,1731422964694' ***** 2024-11-12T14:52:38,722 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T14:52:38,722 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '3945e1ca6f47,35473,1731422964757' ***** 2024-11-12T14:52:38,722 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T14:52:38,723 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T14:52:38,723 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T14:52:38,723 INFO [RS:2;3945e1ca6f47:35473 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T14:52:38,723 INFO [RS:2;3945e1ca6f47:35473 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T14:52:38,723 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T14:52:38,723 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T14:52:38,723 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.HRegionServer(3091): Received CLOSE for d3c298441d993e84c2d9e579914483db 2024-11-12T14:52:38,723 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T14:52:38,724 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T14:52:38,724 INFO [RS:1;3945e1ca6f47:45845 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T14:52:38,724 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T14:52:38,724 INFO [RS:1;3945e1ca6f47:45845 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T14:52:38,724 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(3091): Received CLOSE for a2cc208a6afe25c72a2045fb03e0ecb1 2024-11-12T14:52:38,724 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.HRegionServer(3091): Received CLOSE for ac225c32dbf496a52e546c7e0b1a05af 2024-11-12T14:52:38,724 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.HRegionServer(3091): Received CLOSE for 59fa7126e2ade1f9daaaeb8bd38b09e4 2024-11-12T14:52:38,724 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.HRegionServer(3091): Received CLOSE for 95e5de757651e0aa099998abaa3b598f 2024-11-12T14:52:38,724 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.HRegionServer(3091): Received CLOSE for 489408be5f35a96bee1ee0f68f064599 2024-11-12T14:52:38,724 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.HRegionServer(3091): Received CLOSE for 2cc2e73ccc39f945b27a1d50dd2694bd 2024-11-12T14:52:38,724 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.HRegionServer(3091): Received CLOSE for bb0382e4edc08a7c43f0571deb2374a4 2024-11-12T14:52:38,724 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.HRegionServer(3091): Received CLOSE for 2b04a6d83bd2998d75a1743bdf61651e 2024-11-12T14:52:38,724 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.HRegionServer(3091): Received CLOSE for 92f3e9baa717d57147319661f6f00e06 2024-11-12T14:52:38,724 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.HRegionServer(3091): Received CLOSE for 8b5e68238ca126132dbf55f0fbf38047 2024-11-12T14:52:38,724 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.HRegionServer(3091): Received CLOSE for 9a6f7a226796c8ff7e21a40d463cd27b 2024-11-12T14:52:38,724 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.HRegionServer(3091): Received CLOSE for f65e71cd750425cce9713e30a75c6002 2024-11-12T14:52:38,724 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.HRegionServer(959): stopping server 3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:38,725 INFO [RS:2;3945e1ca6f47:35473 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T14:52:38,725 INFO [RS:2;3945e1ca6f47:35473 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;3945e1ca6f47:35473. 2024-11-12T14:52:38,725 DEBUG [RS:2;3945e1ca6f47:35473 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T14:52:38,725 DEBUG [RS:2;3945e1ca6f47:35473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:52:38,725 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d3c298441d993e84c2d9e579914483db, disabling compactions & flushes 2024-11-12T14:52:38,725 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.HRegionServer(1321): Waiting on 12 regions to close 2024-11-12T14:52:38,725 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db. 2024-11-12T14:52:38,725 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db. 2024-11-12T14:52:38,725 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db. after waiting 0 ms 2024-11-12T14:52:38,725 DEBUG [RS:2;3945e1ca6f47:35473 {}] regionserver.HRegionServer(1351): Waiting on 2b04a6d83bd2998d75a1743bdf61651e, 2cc2e73ccc39f945b27a1d50dd2694bd, 489408be5f35a96bee1ee0f68f064599, 59fa7126e2ade1f9daaaeb8bd38b09e4, 8b5e68238ca126132dbf55f0fbf38047, 92f3e9baa717d57147319661f6f00e06, 95e5de757651e0aa099998abaa3b598f, 9a6f7a226796c8ff7e21a40d463cd27b, ac225c32dbf496a52e546c7e0b1a05af, bb0382e4edc08a7c43f0571deb2374a4, d3c298441d993e84c2d9e579914483db, f65e71cd750425cce9713e30a75c6002 2024-11-12T14:52:38,725 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db. 2024-11-12T14:52:38,725 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(3091): Received CLOSE for efc23c8cc5ebe3a5c96ef5d199223832 2024-11-12T14:52:38,725 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(3091): Received CLOSE for 51b19d6922dec6f0774f3d761593de8f 2024-11-12T14:52:38,725 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(3091): Received CLOSE for aa9c293d6596b8e2756cafba09b3d7dd 2024-11-12T14:52:38,725 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(3091): Received CLOSE for 2d931ca9d93687bd79e96c94cb16690b 2024-11-12T14:52:38,725 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a2cc208a6afe25c72a2045fb03e0ecb1, disabling compactions & flushes 2024-11-12T14:52:38,725 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(3091): Received CLOSE for eaeef83a325d6b259b1b899f552ba8d0 2024-11-12T14:52:38,726 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(3091): Received CLOSE for d7c3d3ae3b942f0414dcbb110be8bc0b 2024-11-12T14:52:38,726 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731422967229.a2cc208a6afe25c72a2045fb03e0ecb1. 2024-11-12T14:52:38,726 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(3091): Received CLOSE for b419710b4e7b088b11da63dff9b7760e 2024-11-12T14:52:38,726 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731422967229.a2cc208a6afe25c72a2045fb03e0ecb1. 2024-11-12T14:52:38,726 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(3091): Received CLOSE for e41036448a3677ed52f58f536690bf38 2024-11-12T14:52:38,726 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(3091): Received CLOSE for f943ce043844f379c1b7ecbf76b5a419 2024-11-12T14:52:38,726 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731422967229.a2cc208a6afe25c72a2045fb03e0ecb1. after waiting 0 ms 2024-11-12T14:52:38,726 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731422967229.a2cc208a6afe25c72a2045fb03e0ecb1. 2024-11-12T14:52:38,726 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(3091): Received CLOSE for df409d6fa4df5d8fd91534d2d90eda60 2024-11-12T14:52:38,726 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(3091): Received CLOSE for 4936dfde0b7af7ea99d36c9e0c24fefc 2024-11-12T14:52:38,726 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(3091): Received CLOSE for 06aa3b2e56ae4404b602d6fa98eabe33 2024-11-12T14:52:38,726 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(3091): Received CLOSE for cf0006d9b8777107f458b7008fc42ba9 2024-11-12T14:52:38,726 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(959): stopping server 3945e1ca6f47,45845,1731422964694 2024-11-12T14:52:38,726 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing a2cc208a6afe25c72a2045fb03e0ecb1 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-12T14:52:38,726 INFO [RS:1;3945e1ca6f47:45845 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T14:52:38,726 INFO [RS:1;3945e1ca6f47:45845 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;3945e1ca6f47:45845. 2024-11-12T14:52:38,726 DEBUG [RS:1;3945e1ca6f47:45845 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T14:52:38,726 DEBUG [RS:1;3945e1ca6f47:45845 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:52:38,726 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T14:52:38,726 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T14:52:38,727 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T14:52:38,727 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-12T14:52:38,727 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(1321): Waiting on 15 regions to close 2024-11-12T14:52:38,727 DEBUG [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(1351): Waiting on 06aa3b2e56ae4404b602d6fa98eabe33, 1588230740, 2d931ca9d93687bd79e96c94cb16690b, 4936dfde0b7af7ea99d36c9e0c24fefc, 51b19d6922dec6f0774f3d761593de8f, a2cc208a6afe25c72a2045fb03e0ecb1, aa9c293d6596b8e2756cafba09b3d7dd, b419710b4e7b088b11da63dff9b7760e, cf0006d9b8777107f458b7008fc42ba9, d7c3d3ae3b942f0414dcbb110be8bc0b, df409d6fa4df5d8fd91534d2d90eda60, e41036448a3677ed52f58f536690bf38, eaeef83a325d6b259b1b899f552ba8d0, efc23c8cc5ebe3a5c96ef5d199223832, f943ce043844f379c1b7ecbf76b5a419 2024-11-12T14:52:38,729 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T14:52:38,729 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T14:52:38,729 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T14:52:38,729 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T14:52:38,729 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T14:52:38,730 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=13.39 KB heapSize=19.70 KB 2024-11-12T14:52:38,731 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d3c298441d993e84c2d9e579914483db/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-12T14:52:38,733 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db. 2024-11-12T14:52:38,733 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d3c298441d993e84c2d9e579914483db: Waiting for close lock at 1731423158724Running coprocessor pre-close hooks at 1731423158725 (+1 ms)Disabling compacts and flushes for region at 1731423158725Disabling writes for close at 1731423158725Writing region close event to WAL at 1731423158727 (+2 ms)Running coprocessor post-close hooks at 1731423158732 (+5 ms)Closed at 1731423158733 (+1 ms) 2024-11-12T14:52:38,734 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,eee,1731422967229.d3c298441d993e84c2d9e579914483db. 2024-11-12T14:52:38,734 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ac225c32dbf496a52e546c7e0b1a05af, disabling compactions & flushes 2024-11-12T14:52:38,734 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731422967229.ac225c32dbf496a52e546c7e0b1a05af. 2024-11-12T14:52:38,734 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731422967229.ac225c32dbf496a52e546c7e0b1a05af. 2024-11-12T14:52:38,734 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731422967229.ac225c32dbf496a52e546c7e0b1a05af. after waiting 0 ms 2024-11-12T14:52:38,734 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731422967229.ac225c32dbf496a52e546c7e0b1a05af. 2024-11-12T14:52:38,734 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing ac225c32dbf496a52e546c7e0b1a05af 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-12T14:52:38,741 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/.tmp/info/973ff944a7dd4506bdc9112f905a4dce is 268, key is testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33./info:regioninfo/1731423156560/Put/seqid=0 2024-11-12T14:52:38,750 INFO [regionserver/3945e1ca6f47:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T14:52:38,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741887_1063 (size=19852) 2024-11-12T14:52:38,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741887_1063 (size=19852) 2024-11-12T14:52:38,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741887_1063 (size=19852) 2024-11-12T14:52:38,755 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a2cc208a6afe25c72a2045fb03e0ecb1/.tmp/colfamily11/3425a94478424fd0a8c7f990ecf823e6 is 52, key is rrr/colfamily11:colfamily11/1731422971678/Put/seqid=0 2024-11-12T14:52:38,756 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.39 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/.tmp/info/973ff944a7dd4506bdc9112f905a4dce 2024-11-12T14:52:38,757 INFO [regionserver/3945e1ca6f47:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T14:52:38,765 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/.tmp/info/973ff944a7dd4506bdc9112f905a4dce as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/info/973ff944a7dd4506bdc9112f905a4dce 2024-11-12T14:52:38,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741888_1064 (size=39263) 2024-11-12T14:52:38,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741888_1064 (size=39263) 2024-11-12T14:52:38,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741888_1064 (size=39263) 2024-11-12T14:52:38,768 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac225c32dbf496a52e546c7e0b1a05af/.tmp/colfamily11/d477742b3fc24297a80ff9fbcadc6934 is 52, key is mmm/colfamily11:colfamily11/1731422971559/Put/seqid=0 2024-11-12T14:52:38,770 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a2cc208a6afe25c72a2045fb03e0ecb1/.tmp/colfamily11/3425a94478424fd0a8c7f990ecf823e6 2024-11-12T14:52:38,775 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/info/973ff944a7dd4506bdc9112f905a4dce, entries=72, sequenceid=81, filesize=19.4 K 2024-11-12T14:52:38,776 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~13.39 KB/13710, heapSize ~18.94 KB/19392, currentSize=0 B/0 for 1588230740 in 47ms, sequenceid=81, compaction requested=false 2024-11-12T14:52:38,778 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a2cc208a6afe25c72a2045fb03e0ecb1/.tmp/colfamily11/3425a94478424fd0a8c7f990ecf823e6 as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a2cc208a6afe25c72a2045fb03e0ecb1/colfamily11/3425a94478424fd0a8c7f990ecf823e6 2024-11-12T14:52:38,780 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/hbase/meta/1588230740/recovered.edits/84.seqid, newMaxSeqId=84, maxSeqId=60 2024-11-12T14:52:38,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741889_1065 (size=39263) 2024-11-12T14:52:38,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741889_1065 (size=39263) 2024-11-12T14:52:38,782 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T14:52:38,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741889_1065 (size=39263) 2024-11-12T14:52:38,782 INFO [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T14:52:38,782 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731423158727Running coprocessor pre-close hooks at 1731423158729 (+2 ms)Disabling compacts and flushes for region at 1731423158729Disabling writes for close at 1731423158729Obtaining lock to block concurrent updates at 1731423158730 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731423158730Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=13710, getHeapSize=20112, getOffHeapSize=0, getCellsCount=72 at 1731423158735 (+5 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731423158736 (+1 ms)Flushing 1588230740/info: creating writer at 1731423158736Flushing 1588230740/info: appending metadata at 1731423158741 (+5 ms)Flushing 1588230740/info: closing flushed file at 1731423158741Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b1e7d30: reopening flushed file at 1731423158763 (+22 ms)Finished flush of dataSize ~13.39 KB/13710, heapSize ~18.94 KB/19392, currentSize=0 B/0 for 1588230740 in 47ms, sequenceid=81, compaction requested=false at 1731423158776 (+13 ms)Writing region close event to WAL at 1731423158777 (+1 ms)Running coprocessor post-close hooks at 1731423158782 (+5 ms)Closed at 1731423158782 2024-11-12T14:52:38,782 DEBUG [RS_CLOSE_META-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-12T14:52:38,782 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac225c32dbf496a52e546c7e0b1a05af/.tmp/colfamily11/d477742b3fc24297a80ff9fbcadc6934 2024-11-12T14:52:38,786 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a2cc208a6afe25c72a2045fb03e0ecb1/colfamily11/3425a94478424fd0a8c7f990ecf823e6, entries=703, sequenceid=5, filesize=38.3 K 2024-11-12T14:52:38,787 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for a2cc208a6afe25c72a2045fb03e0ecb1 in 61ms, sequenceid=5, compaction requested=false 2024-11-12T14:52:38,790 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac225c32dbf496a52e546c7e0b1a05af/.tmp/colfamily11/d477742b3fc24297a80ff9fbcadc6934 as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac225c32dbf496a52e546c7e0b1a05af/colfamily11/d477742b3fc24297a80ff9fbcadc6934 2024-11-12T14:52:38,791 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/a2cc208a6afe25c72a2045fb03e0ecb1/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-12T14:52:38,792 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731422967229.a2cc208a6afe25c72a2045fb03e0ecb1. 2024-11-12T14:52:38,792 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a2cc208a6afe25c72a2045fb03e0ecb1: Waiting for close lock at 1731423158725Running coprocessor pre-close hooks at 1731423158725Disabling compacts and flushes for region at 1731423158725Disabling writes for close at 1731423158726 (+1 ms)Obtaining lock to block concurrent updates at 1731423158726Preparing flush snapshotting stores in a2cc208a6afe25c72a2045fb03e0ecb1 at 1731423158726Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731422967229.a2cc208a6afe25c72a2045fb03e0ecb1., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731423158735 (+9 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731422967229.a2cc208a6afe25c72a2045fb03e0ecb1. at 1731423158736 (+1 ms)Flushing a2cc208a6afe25c72a2045fb03e0ecb1/colfamily11: creating writer at 1731423158736Flushing a2cc208a6afe25c72a2045fb03e0ecb1/colfamily11: appending metadata at 1731423158755 (+19 ms)Flushing a2cc208a6afe25c72a2045fb03e0ecb1/colfamily11: closing flushed file at 1731423158755Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11519a2c: reopening flushed file at 1731423158777 (+22 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for a2cc208a6afe25c72a2045fb03e0ecb1 in 61ms, sequenceid=5, compaction requested=false at 1731423158787 (+10 ms)Writing region close event to WAL at 1731423158788 (+1 ms)Running coprocessor post-close hooks at 1731423158792 (+4 ms)Closed at 1731423158792 2024-11-12T14:52:38,792 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,rrr,1731422967229.a2cc208a6afe25c72a2045fb03e0ecb1. 2024-11-12T14:52:38,793 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing efc23c8cc5ebe3a5c96ef5d199223832, disabling compactions & flushes 2024-11-12T14:52:38,793 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731422967229.efc23c8cc5ebe3a5c96ef5d199223832. 2024-11-12T14:52:38,793 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731422967229.efc23c8cc5ebe3a5c96ef5d199223832. 2024-11-12T14:52:38,793 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731422967229.efc23c8cc5ebe3a5c96ef5d199223832. after waiting 0 ms 2024-11-12T14:52:38,793 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731422967229.efc23c8cc5ebe3a5c96ef5d199223832. 2024-11-12T14:52:38,793 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing efc23c8cc5ebe3a5c96ef5d199223832 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-12T14:52:38,796 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac225c32dbf496a52e546c7e0b1a05af/colfamily11/d477742b3fc24297a80ff9fbcadc6934, entries=703, sequenceid=5, filesize=38.3 K 2024-11-12T14:52:38,798 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for ac225c32dbf496a52e546c7e0b1a05af in 63ms, sequenceid=5, compaction requested=false 2024-11-12T14:52:38,802 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/ac225c32dbf496a52e546c7e0b1a05af/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-12T14:52:38,803 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731422967229.ac225c32dbf496a52e546c7e0b1a05af. 2024-11-12T14:52:38,803 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ac225c32dbf496a52e546c7e0b1a05af: Waiting for close lock at 1731423158734Running coprocessor pre-close hooks at 1731423158734Disabling compacts and flushes for region at 1731423158734Disabling writes for close at 1731423158734Obtaining lock to block concurrent updates at 1731423158734Preparing flush snapshotting stores in ac225c32dbf496a52e546c7e0b1a05af at 1731423158734Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731422967229.ac225c32dbf496a52e546c7e0b1a05af., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731423158735 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731422967229.ac225c32dbf496a52e546c7e0b1a05af. at 1731423158736 (+1 ms)Flushing ac225c32dbf496a52e546c7e0b1a05af/colfamily11: creating writer at 1731423158736Flushing ac225c32dbf496a52e546c7e0b1a05af/colfamily11: appending metadata at 1731423158767 (+31 ms)Flushing ac225c32dbf496a52e546c7e0b1a05af/colfamily11: closing flushed file at 1731423158767Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27228e12: reopening flushed file at 1731423158789 (+22 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for ac225c32dbf496a52e546c7e0b1a05af in 63ms, sequenceid=5, compaction requested=false at 1731423158798 (+9 ms)Writing region close event to WAL at 1731423158798Running coprocessor post-close hooks at 1731423158803 (+5 ms)Closed at 1731423158803 2024-11-12T14:52:38,803 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,mmm,1731422967229.ac225c32dbf496a52e546c7e0b1a05af. 2024-11-12T14:52:38,803 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 59fa7126e2ade1f9daaaeb8bd38b09e4, disabling compactions & flushes 2024-11-12T14:52:38,803 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731422967229.59fa7126e2ade1f9daaaeb8bd38b09e4. 2024-11-12T14:52:38,803 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731422967229.59fa7126e2ade1f9daaaeb8bd38b09e4. 2024-11-12T14:52:38,803 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731422967229.59fa7126e2ade1f9daaaeb8bd38b09e4. after waiting 0 ms 2024-11-12T14:52:38,803 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731422967229.59fa7126e2ade1f9daaaeb8bd38b09e4. 2024-11-12T14:52:38,804 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 59fa7126e2ade1f9daaaeb8bd38b09e4 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-12T14:52:38,812 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/efc23c8cc5ebe3a5c96ef5d199223832/.tmp/colfamily11/a75dbedca5e546629d8d4a186d610683 is 52, key is uuu/colfamily11:colfamily11/1731422971725/Put/seqid=0 2024-11-12T14:52:38,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741890_1066 (size=39263) 2024-11-12T14:52:38,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741890_1066 (size=39263) 2024-11-12T14:52:38,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741890_1066 (size=39263) 2024-11-12T14:52:38,822 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/efc23c8cc5ebe3a5c96ef5d199223832/.tmp/colfamily11/a75dbedca5e546629d8d4a186d610683 2024-11-12T14:52:38,826 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/59fa7126e2ade1f9daaaeb8bd38b09e4/.tmp/colfamily11/23142f46d12f476eb5cac25a8ee63aed is 52, key is fff/colfamily11:colfamily11/1731422971448/Put/seqid=0 2024-11-12T14:52:38,830 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/efc23c8cc5ebe3a5c96ef5d199223832/.tmp/colfamily11/a75dbedca5e546629d8d4a186d610683 as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/efc23c8cc5ebe3a5c96ef5d199223832/colfamily11/a75dbedca5e546629d8d4a186d610683 2024-11-12T14:52:38,838 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/efc23c8cc5ebe3a5c96ef5d199223832/colfamily11/a75dbedca5e546629d8d4a186d610683, entries=703, sequenceid=5, filesize=38.3 K 2024-11-12T14:52:38,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741891_1067 (size=39263) 2024-11-12T14:52:38,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741891_1067 (size=39263) 2024-11-12T14:52:38,839 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for efc23c8cc5ebe3a5c96ef5d199223832 in 46ms, sequenceid=5, compaction requested=false 2024-11-12T14:52:38,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741891_1067 (size=39263) 2024-11-12T14:52:38,841 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/59fa7126e2ade1f9daaaeb8bd38b09e4/.tmp/colfamily11/23142f46d12f476eb5cac25a8ee63aed 2024-11-12T14:52:38,845 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/efc23c8cc5ebe3a5c96ef5d199223832/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-12T14:52:38,846 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731422967229.efc23c8cc5ebe3a5c96ef5d199223832. 2024-11-12T14:52:38,846 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for efc23c8cc5ebe3a5c96ef5d199223832: Waiting for close lock at 1731423158793Running coprocessor pre-close hooks at 1731423158793Disabling compacts and flushes for region at 1731423158793Disabling writes for close at 1731423158793Obtaining lock to block concurrent updates at 1731423158793Preparing flush snapshotting stores in efc23c8cc5ebe3a5c96ef5d199223832 at 1731423158793Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731422967229.efc23c8cc5ebe3a5c96ef5d199223832., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731423158793Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731422967229.efc23c8cc5ebe3a5c96ef5d199223832. at 1731423158794 (+1 ms)Flushing efc23c8cc5ebe3a5c96ef5d199223832/colfamily11: creating writer at 1731423158794Flushing efc23c8cc5ebe3a5c96ef5d199223832/colfamily11: appending metadata at 1731423158812 (+18 ms)Flushing efc23c8cc5ebe3a5c96ef5d199223832/colfamily11: closing flushed file at 1731423158812Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d16081e: reopening flushed file at 1731423158829 (+17 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for efc23c8cc5ebe3a5c96ef5d199223832 in 46ms, sequenceid=5, compaction requested=false at 1731423158839 (+10 ms)Writing region close event to WAL at 1731423158841 (+2 ms)Running coprocessor post-close hooks at 1731423158846 (+5 ms)Closed at 1731423158846 2024-11-12T14:52:38,847 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,uuu,1731422967229.efc23c8cc5ebe3a5c96ef5d199223832. 2024-11-12T14:52:38,847 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 51b19d6922dec6f0774f3d761593de8f, disabling compactions & flushes 2024-11-12T14:52:38,847 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731422967229.51b19d6922dec6f0774f3d761593de8f. 2024-11-12T14:52:38,847 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731422967229.51b19d6922dec6f0774f3d761593de8f. 2024-11-12T14:52:38,847 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731422967229.51b19d6922dec6f0774f3d761593de8f. after waiting 0 ms 2024-11-12T14:52:38,847 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731422967229.51b19d6922dec6f0774f3d761593de8f. 2024-11-12T14:52:38,847 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 51b19d6922dec6f0774f3d761593de8f 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-12T14:52:38,850 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/59fa7126e2ade1f9daaaeb8bd38b09e4/.tmp/colfamily11/23142f46d12f476eb5cac25a8ee63aed as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/59fa7126e2ade1f9daaaeb8bd38b09e4/colfamily11/23142f46d12f476eb5cac25a8ee63aed 2024-11-12T14:52:38,858 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/59fa7126e2ade1f9daaaeb8bd38b09e4/colfamily11/23142f46d12f476eb5cac25a8ee63aed, entries=703, sequenceid=5, filesize=38.3 K 2024-11-12T14:52:38,859 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 59fa7126e2ade1f9daaaeb8bd38b09e4 in 56ms, sequenceid=5, compaction requested=false 2024-11-12T14:52:38,864 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/59fa7126e2ade1f9daaaeb8bd38b09e4/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-12T14:52:38,865 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731422967229.59fa7126e2ade1f9daaaeb8bd38b09e4. 2024-11-12T14:52:38,865 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 59fa7126e2ade1f9daaaeb8bd38b09e4: Waiting for close lock at 1731423158803Running coprocessor pre-close hooks at 1731423158803Disabling compacts and flushes for region at 1731423158803Disabling writes for close at 1731423158803Obtaining lock to block concurrent updates at 1731423158804 (+1 ms)Preparing flush snapshotting stores in 59fa7126e2ade1f9daaaeb8bd38b09e4 at 1731423158804Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731422967229.59fa7126e2ade1f9daaaeb8bd38b09e4., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731423158804Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731422967229.59fa7126e2ade1f9daaaeb8bd38b09e4. at 1731423158804Flushing 59fa7126e2ade1f9daaaeb8bd38b09e4/colfamily11: creating writer at 1731423158804Flushing 59fa7126e2ade1f9daaaeb8bd38b09e4/colfamily11: appending metadata at 1731423158826 (+22 ms)Flushing 59fa7126e2ade1f9daaaeb8bd38b09e4/colfamily11: closing flushed file at 1731423158826Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@502d6804: reopening flushed file at 1731423158849 (+23 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 59fa7126e2ade1f9daaaeb8bd38b09e4 in 56ms, sequenceid=5, compaction requested=false at 1731423158859 (+10 ms)Writing region close event to WAL at 1731423158860 (+1 ms)Running coprocessor post-close hooks at 1731423158865 (+5 ms)Closed at 1731423158865 2024-11-12T14:52:38,865 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,fff,1731422967229.59fa7126e2ade1f9daaaeb8bd38b09e4. 2024-11-12T14:52:38,866 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 95e5de757651e0aa099998abaa3b598f, disabling compactions & flushes 2024-11-12T14:52:38,866 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731422967229.95e5de757651e0aa099998abaa3b598f. 2024-11-12T14:52:38,866 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731422967229.95e5de757651e0aa099998abaa3b598f. 2024-11-12T14:52:38,866 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731422967229.95e5de757651e0aa099998abaa3b598f. after waiting 0 ms 2024-11-12T14:52:38,866 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731422967229.95e5de757651e0aa099998abaa3b598f. 2024-11-12T14:52:38,866 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 95e5de757651e0aa099998abaa3b598f 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-12T14:52:38,867 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/51b19d6922dec6f0774f3d761593de8f/.tmp/colfamily11/87829ec680e44a63ae1ac8b29bb4dbd5 is 52, key is lll/colfamily11:colfamily11/1731422971597/Put/seqid=0 2024-11-12T14:52:38,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741892_1068 (size=39263) 2024-11-12T14:52:38,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741892_1068 (size=39263) 2024-11-12T14:52:38,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741892_1068 (size=39263) 2024-11-12T14:52:38,883 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/51b19d6922dec6f0774f3d761593de8f/.tmp/colfamily11/87829ec680e44a63ae1ac8b29bb4dbd5 2024-11-12T14:52:38,890 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/95e5de757651e0aa099998abaa3b598f/.tmp/colfamily11/ffd1b8c2b8ea40a9b6259866c5438e9c is 52, key is yyy/colfamily11:colfamily11/1731422971722/Put/seqid=0 2024-11-12T14:52:38,891 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/51b19d6922dec6f0774f3d761593de8f/.tmp/colfamily11/87829ec680e44a63ae1ac8b29bb4dbd5 as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/51b19d6922dec6f0774f3d761593de8f/colfamily11/87829ec680e44a63ae1ac8b29bb4dbd5 2024-11-12T14:52:38,899 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/51b19d6922dec6f0774f3d761593de8f/colfamily11/87829ec680e44a63ae1ac8b29bb4dbd5, entries=703, sequenceid=5, filesize=38.3 K 2024-11-12T14:52:38,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741893_1069 (size=39263) 2024-11-12T14:52:38,900 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 51b19d6922dec6f0774f3d761593de8f in 53ms, sequenceid=5, compaction requested=false 2024-11-12T14:52:38,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741893_1069 (size=39263) 2024-11-12T14:52:38,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741893_1069 (size=39263) 2024-11-12T14:52:38,903 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/95e5de757651e0aa099998abaa3b598f/.tmp/colfamily11/ffd1b8c2b8ea40a9b6259866c5438e9c 2024-11-12T14:52:38,906 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/51b19d6922dec6f0774f3d761593de8f/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-12T14:52:38,907 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731422967229.51b19d6922dec6f0774f3d761593de8f. 2024-11-12T14:52:38,907 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 51b19d6922dec6f0774f3d761593de8f: Waiting for close lock at 1731423158847Running coprocessor pre-close hooks at 1731423158847Disabling compacts and flushes for region at 1731423158847Disabling writes for close at 1731423158847Obtaining lock to block concurrent updates at 1731423158847Preparing flush snapshotting stores in 51b19d6922dec6f0774f3d761593de8f at 1731423158847Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731422967229.51b19d6922dec6f0774f3d761593de8f., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731423158848 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731422967229.51b19d6922dec6f0774f3d761593de8f. at 1731423158848Flushing 51b19d6922dec6f0774f3d761593de8f/colfamily11: creating writer at 1731423158848Flushing 51b19d6922dec6f0774f3d761593de8f/colfamily11: appending metadata at 1731423158867 (+19 ms)Flushing 51b19d6922dec6f0774f3d761593de8f/colfamily11: closing flushed file at 1731423158867Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d973fbf: reopening flushed file at 1731423158890 (+23 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 51b19d6922dec6f0774f3d761593de8f in 53ms, sequenceid=5, compaction requested=false at 1731423158900 (+10 ms)Writing region close event to WAL at 1731423158902 (+2 ms)Running coprocessor post-close hooks at 1731423158907 (+5 ms)Closed at 1731423158907 2024-11-12T14:52:38,907 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,lll,1731422967229.51b19d6922dec6f0774f3d761593de8f. 2024-11-12T14:52:38,908 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing aa9c293d6596b8e2756cafba09b3d7dd, disabling compactions & flushes 2024-11-12T14:52:38,908 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731422967229.aa9c293d6596b8e2756cafba09b3d7dd. 2024-11-12T14:52:38,908 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731422967229.aa9c293d6596b8e2756cafba09b3d7dd. 2024-11-12T14:52:38,908 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731422967229.aa9c293d6596b8e2756cafba09b3d7dd. after waiting 0 ms 2024-11-12T14:52:38,908 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731422967229.aa9c293d6596b8e2756cafba09b3d7dd. 2024-11-12T14:52:38,908 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing aa9c293d6596b8e2756cafba09b3d7dd 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-12T14:52:38,912 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/95e5de757651e0aa099998abaa3b598f/.tmp/colfamily11/ffd1b8c2b8ea40a9b6259866c5438e9c as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/95e5de757651e0aa099998abaa3b598f/colfamily11/ffd1b8c2b8ea40a9b6259866c5438e9c 2024-11-12T14:52:38,921 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/95e5de757651e0aa099998abaa3b598f/colfamily11/ffd1b8c2b8ea40a9b6259866c5438e9c, entries=703, sequenceid=5, filesize=38.3 K 2024-11-12T14:52:38,922 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 95e5de757651e0aa099998abaa3b598f in 56ms, sequenceid=5, compaction requested=false 2024-11-12T14:52:38,925 DEBUG [RS:2;3945e1ca6f47:35473 {}] regionserver.HRegionServer(1351): Waiting on 2b04a6d83bd2998d75a1743bdf61651e, 2cc2e73ccc39f945b27a1d50dd2694bd, 489408be5f35a96bee1ee0f68f064599, 8b5e68238ca126132dbf55f0fbf38047, 92f3e9baa717d57147319661f6f00e06, 95e5de757651e0aa099998abaa3b598f, 9a6f7a226796c8ff7e21a40d463cd27b, bb0382e4edc08a7c43f0571deb2374a4, f65e71cd750425cce9713e30a75c6002 2024-11-12T14:52:38,926 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/95e5de757651e0aa099998abaa3b598f/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-12T14:52:38,927 DEBUG [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(1351): Waiting on 06aa3b2e56ae4404b602d6fa98eabe33, 2d931ca9d93687bd79e96c94cb16690b, 4936dfde0b7af7ea99d36c9e0c24fefc, aa9c293d6596b8e2756cafba09b3d7dd, b419710b4e7b088b11da63dff9b7760e, cf0006d9b8777107f458b7008fc42ba9, d7c3d3ae3b942f0414dcbb110be8bc0b, df409d6fa4df5d8fd91534d2d90eda60, e41036448a3677ed52f58f536690bf38, eaeef83a325d6b259b1b899f552ba8d0, f943ce043844f379c1b7ecbf76b5a419 2024-11-12T14:52:38,927 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731422967229.95e5de757651e0aa099998abaa3b598f. 2024-11-12T14:52:38,927 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 95e5de757651e0aa099998abaa3b598f: Waiting for close lock at 1731423158866Running coprocessor pre-close hooks at 1731423158866Disabling compacts and flushes for region at 1731423158866Disabling writes for close at 1731423158866Obtaining lock to block concurrent updates at 1731423158866Preparing flush snapshotting stores in 95e5de757651e0aa099998abaa3b598f at 1731423158866Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731422967229.95e5de757651e0aa099998abaa3b598f., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731423158866Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731422967229.95e5de757651e0aa099998abaa3b598f. at 1731423158867 (+1 ms)Flushing 95e5de757651e0aa099998abaa3b598f/colfamily11: creating writer at 1731423158867Flushing 95e5de757651e0aa099998abaa3b598f/colfamily11: appending metadata at 1731423158889 (+22 ms)Flushing 95e5de757651e0aa099998abaa3b598f/colfamily11: closing flushed file at 1731423158889Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31437b5e: reopening flushed file at 1731423158911 (+22 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 95e5de757651e0aa099998abaa3b598f in 56ms, sequenceid=5, compaction requested=false at 1731423158922 (+11 ms)Writing region close event to WAL at 1731423158923 (+1 ms)Running coprocessor post-close hooks at 1731423158927 (+4 ms)Closed at 1731423158927 2024-11-12T14:52:38,927 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,yyy,1731422967229.95e5de757651e0aa099998abaa3b598f. 2024-11-12T14:52:38,928 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 489408be5f35a96bee1ee0f68f064599, disabling compactions & flushes 2024-11-12T14:52:38,928 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731422967229.489408be5f35a96bee1ee0f68f064599. 2024-11-12T14:52:38,928 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731422967229.489408be5f35a96bee1ee0f68f064599. 2024-11-12T14:52:38,928 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731422967229.489408be5f35a96bee1ee0f68f064599. after waiting 0 ms 2024-11-12T14:52:38,928 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731422967229.489408be5f35a96bee1ee0f68f064599. 2024-11-12T14:52:38,928 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 489408be5f35a96bee1ee0f68f064599 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-12T14:52:38,931 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa9c293d6596b8e2756cafba09b3d7dd/.tmp/colfamily11/f27aec58de504798b7a54a3abb920405 is 52, key is xxx/colfamily11:colfamily11/1731422971808/Put/seqid=0 2024-11-12T14:52:38,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741894_1070 (size=39263) 2024-11-12T14:52:38,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741894_1070 (size=39263) 2024-11-12T14:52:38,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741894_1070 (size=39263) 2024-11-12T14:52:38,945 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa9c293d6596b8e2756cafba09b3d7dd/.tmp/colfamily11/f27aec58de504798b7a54a3abb920405 2024-11-12T14:52:38,952 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/489408be5f35a96bee1ee0f68f064599/.tmp/colfamily11/55aa1ae36b604436a71c712786a74818 is 52, key is bbb/colfamily11:colfamily11/1731422971355/Put/seqid=0 2024-11-12T14:52:38,957 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa9c293d6596b8e2756cafba09b3d7dd/.tmp/colfamily11/f27aec58de504798b7a54a3abb920405 as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa9c293d6596b8e2756cafba09b3d7dd/colfamily11/f27aec58de504798b7a54a3abb920405 2024-11-12T14:52:38,965 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa9c293d6596b8e2756cafba09b3d7dd/colfamily11/f27aec58de504798b7a54a3abb920405, entries=703, sequenceid=5, filesize=38.3 K 2024-11-12T14:52:38,966 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for aa9c293d6596b8e2756cafba09b3d7dd in 58ms, sequenceid=5, compaction requested=false 2024-11-12T14:52:38,970 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/aa9c293d6596b8e2756cafba09b3d7dd/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-12T14:52:38,971 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731422967229.aa9c293d6596b8e2756cafba09b3d7dd. 2024-11-12T14:52:38,971 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for aa9c293d6596b8e2756cafba09b3d7dd: Waiting for close lock at 1731423158908Running coprocessor pre-close hooks at 1731423158908Disabling compacts and flushes for region at 1731423158908Disabling writes for close at 1731423158908Obtaining lock to block concurrent updates at 1731423158908Preparing flush snapshotting stores in aa9c293d6596b8e2756cafba09b3d7dd at 1731423158908Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731422967229.aa9c293d6596b8e2756cafba09b3d7dd., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731423158908Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731422967229.aa9c293d6596b8e2756cafba09b3d7dd. at 1731423158909 (+1 ms)Flushing aa9c293d6596b8e2756cafba09b3d7dd/colfamily11: creating writer at 1731423158909Flushing aa9c293d6596b8e2756cafba09b3d7dd/colfamily11: appending metadata at 1731423158931 (+22 ms)Flushing aa9c293d6596b8e2756cafba09b3d7dd/colfamily11: closing flushed file at 1731423158931Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55e2f339: reopening flushed file at 1731423158953 (+22 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for aa9c293d6596b8e2756cafba09b3d7dd in 58ms, sequenceid=5, compaction requested=false at 1731423158966 (+13 ms)Writing region close event to WAL at 1731423158967 (+1 ms)Running coprocessor post-close hooks at 1731423158971 (+4 ms)Closed at 1731423158971 2024-11-12T14:52:38,971 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,xxx,1731422967229.aa9c293d6596b8e2756cafba09b3d7dd. 2024-11-12T14:52:38,971 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2d931ca9d93687bd79e96c94cb16690b, disabling compactions & flushes 2024-11-12T14:52:38,971 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b. 2024-11-12T14:52:38,971 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b. 2024-11-12T14:52:38,971 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b. after waiting 0 ms 2024-11-12T14:52:38,972 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b. 2024-11-12T14:52:38,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741895_1071 (size=39263) 2024-11-12T14:52:38,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741895_1071 (size=39263) 2024-11-12T14:52:38,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741895_1071 (size=39263) 2024-11-12T14:52:38,978 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/489408be5f35a96bee1ee0f68f064599/.tmp/colfamily11/55aa1ae36b604436a71c712786a74818 2024-11-12T14:52:38,987 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2d931ca9d93687bd79e96c94cb16690b/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-12T14:52:38,988 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b. 2024-11-12T14:52:38,989 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2d931ca9d93687bd79e96c94cb16690b: Waiting for close lock at 1731423158971Running coprocessor pre-close hooks at 1731423158971Disabling compacts and flushes for region at 1731423158971Disabling writes for close at 1731423158972 (+1 ms)Writing region close event to WAL at 1731423158975 (+3 ms)Running coprocessor post-close hooks at 1731423158988 (+13 ms)Closed at 1731423158988 2024-11-12T14:52:38,989 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ttt,1731422967229.2d931ca9d93687bd79e96c94cb16690b. 2024-11-12T14:52:38,989 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eaeef83a325d6b259b1b899f552ba8d0, disabling compactions & flushes 2024-11-12T14:52:38,989 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731422967229.eaeef83a325d6b259b1b899f552ba8d0. 2024-11-12T14:52:38,989 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731422967229.eaeef83a325d6b259b1b899f552ba8d0. 2024-11-12T14:52:38,989 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731422967229.eaeef83a325d6b259b1b899f552ba8d0. after waiting 0 ms 2024-11-12T14:52:38,989 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731422967229.eaeef83a325d6b259b1b899f552ba8d0. 2024-11-12T14:52:38,989 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing eaeef83a325d6b259b1b899f552ba8d0 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-12T14:52:38,994 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/489408be5f35a96bee1ee0f68f064599/.tmp/colfamily11/55aa1ae36b604436a71c712786a74818 as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/489408be5f35a96bee1ee0f68f064599/colfamily11/55aa1ae36b604436a71c712786a74818 2024-11-12T14:52:39,001 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/489408be5f35a96bee1ee0f68f064599/colfamily11/55aa1ae36b604436a71c712786a74818, entries=703, sequenceid=5, filesize=38.3 K 2024-11-12T14:52:39,002 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 489408be5f35a96bee1ee0f68f064599 in 73ms, sequenceid=5, compaction requested=false 2024-11-12T14:52:39,015 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/489408be5f35a96bee1ee0f68f064599/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-12T14:52:39,015 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731422967229.489408be5f35a96bee1ee0f68f064599. 2024-11-12T14:52:39,016 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 489408be5f35a96bee1ee0f68f064599: Waiting for close lock at 1731423158928Running coprocessor pre-close hooks at 1731423158928Disabling compacts and flushes for region at 1731423158928Disabling writes for close at 1731423158928Obtaining lock to block concurrent updates at 1731423158928Preparing flush snapshotting stores in 489408be5f35a96bee1ee0f68f064599 at 1731423158928Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731422967229.489408be5f35a96bee1ee0f68f064599., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731423158928Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731422967229.489408be5f35a96bee1ee0f68f064599. at 1731423158929 (+1 ms)Flushing 489408be5f35a96bee1ee0f68f064599/colfamily11: creating writer at 1731423158929Flushing 489408be5f35a96bee1ee0f68f064599/colfamily11: appending metadata at 1731423158951 (+22 ms)Flushing 489408be5f35a96bee1ee0f68f064599/colfamily11: closing flushed file at 1731423158951Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53d1f3cf: reopening flushed file at 1731423158992 (+41 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 489408be5f35a96bee1ee0f68f064599 in 73ms, sequenceid=5, compaction requested=false at 1731423159002 (+10 ms)Writing region close event to WAL at 1731423159007 (+5 ms)Running coprocessor post-close hooks at 1731423159015 (+8 ms)Closed at 1731423159015 2024-11-12T14:52:39,016 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,bbb,1731422967229.489408be5f35a96bee1ee0f68f064599. 2024-11-12T14:52:39,016 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2cc2e73ccc39f945b27a1d50dd2694bd, disabling compactions & flushes 2024-11-12T14:52:39,016 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd. 2024-11-12T14:52:39,016 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd. 2024-11-12T14:52:39,016 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd. after waiting 0 ms 2024-11-12T14:52:39,016 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd. 2024-11-12T14:52:39,017 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/eaeef83a325d6b259b1b899f552ba8d0/.tmp/colfamily11/287ac62aff54438087a23e5755a48310 is 52, key is kkk/colfamily11:colfamily11/1731422971558/Put/seqid=0 2024-11-12T14:52:39,034 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2cc2e73ccc39f945b27a1d50dd2694bd/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-12T14:52:39,035 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd. 2024-11-12T14:52:39,035 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2cc2e73ccc39f945b27a1d50dd2694bd: Waiting for close lock at 1731423159016Running coprocessor pre-close hooks at 1731423159016Disabling compacts and flushes for region at 1731423159016Disabling writes for close at 1731423159016Writing region close event to WAL at 1731423159024 (+8 ms)Running coprocessor post-close hooks at 1731423159034 (+10 ms)Closed at 1731423159034 2024-11-12T14:52:39,035 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,qqq,1731422967229.2cc2e73ccc39f945b27a1d50dd2694bd. 2024-11-12T14:52:39,035 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bb0382e4edc08a7c43f0571deb2374a4, disabling compactions & flushes 2024-11-12T14:52:39,035 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731422967229.bb0382e4edc08a7c43f0571deb2374a4. 2024-11-12T14:52:39,035 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731422967229.bb0382e4edc08a7c43f0571deb2374a4. 2024-11-12T14:52:39,035 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731422967229.bb0382e4edc08a7c43f0571deb2374a4. after waiting 0 ms 2024-11-12T14:52:39,035 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731422967229.bb0382e4edc08a7c43f0571deb2374a4. 2024-11-12T14:52:39,036 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing bb0382e4edc08a7c43f0571deb2374a4 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-12T14:52:39,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741896_1072 (size=39263) 2024-11-12T14:52:39,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741896_1072 (size=39263) 2024-11-12T14:52:39,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741896_1072 (size=39263) 2024-11-12T14:52:39,039 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/eaeef83a325d6b259b1b899f552ba8d0/.tmp/colfamily11/287ac62aff54438087a23e5755a48310 2024-11-12T14:52:39,058 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bb0382e4edc08a7c43f0571deb2374a4/.tmp/colfamily11/b6d28e100c1a43e8b426cbf04316f02c is 52, key is ooo/colfamily11:colfamily11/1731422971600/Put/seqid=0 2024-11-12T14:52:39,059 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/eaeef83a325d6b259b1b899f552ba8d0/.tmp/colfamily11/287ac62aff54438087a23e5755a48310 as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/eaeef83a325d6b259b1b899f552ba8d0/colfamily11/287ac62aff54438087a23e5755a48310 2024-11-12T14:52:39,070 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/eaeef83a325d6b259b1b899f552ba8d0/colfamily11/287ac62aff54438087a23e5755a48310, entries=703, sequenceid=5, filesize=38.3 K 2024-11-12T14:52:39,073 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for eaeef83a325d6b259b1b899f552ba8d0 in 84ms, sequenceid=5, compaction requested=false 2024-11-12T14:52:39,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741897_1073 (size=39263) 2024-11-12T14:52:39,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741897_1073 (size=39263) 2024-11-12T14:52:39,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741897_1073 (size=39263) 2024-11-12T14:52:39,082 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bb0382e4edc08a7c43f0571deb2374a4/.tmp/colfamily11/b6d28e100c1a43e8b426cbf04316f02c 2024-11-12T14:52:39,089 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/eaeef83a325d6b259b1b899f552ba8d0/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-12T14:52:39,090 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731422967229.eaeef83a325d6b259b1b899f552ba8d0. 2024-11-12T14:52:39,090 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eaeef83a325d6b259b1b899f552ba8d0: Waiting for close lock at 1731423158989Running coprocessor pre-close hooks at 1731423158989Disabling compacts and flushes for region at 1731423158989Disabling writes for close at 1731423158989Obtaining lock to block concurrent updates at 1731423158989Preparing flush snapshotting stores in eaeef83a325d6b259b1b899f552ba8d0 at 1731423158989Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731422967229.eaeef83a325d6b259b1b899f552ba8d0., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731423158990 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731422967229.eaeef83a325d6b259b1b899f552ba8d0. at 1731423158991 (+1 ms)Flushing eaeef83a325d6b259b1b899f552ba8d0/colfamily11: creating writer at 1731423158991Flushing eaeef83a325d6b259b1b899f552ba8d0/colfamily11: appending metadata at 1731423159017 (+26 ms)Flushing eaeef83a325d6b259b1b899f552ba8d0/colfamily11: closing flushed file at 1731423159017Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fbc5164: reopening flushed file at 1731423159057 (+40 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for eaeef83a325d6b259b1b899f552ba8d0 in 84ms, sequenceid=5, compaction requested=false at 1731423159073 (+16 ms)Writing region close event to WAL at 1731423159078 (+5 ms)Running coprocessor post-close hooks at 1731423159090 (+12 ms)Closed at 1731423159090 2024-11-12T14:52:39,090 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,kkk,1731422967229.eaeef83a325d6b259b1b899f552ba8d0. 2024-11-12T14:52:39,091 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d7c3d3ae3b942f0414dcbb110be8bc0b, disabling compactions & flushes 2024-11-12T14:52:39,091 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b. 2024-11-12T14:52:39,091 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b. 2024-11-12T14:52:39,091 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b. after waiting 0 ms 2024-11-12T14:52:39,091 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b. 2024-11-12T14:52:39,093 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bb0382e4edc08a7c43f0571deb2374a4/.tmp/colfamily11/b6d28e100c1a43e8b426cbf04316f02c as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bb0382e4edc08a7c43f0571deb2374a4/colfamily11/b6d28e100c1a43e8b426cbf04316f02c 2024-11-12T14:52:39,101 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bb0382e4edc08a7c43f0571deb2374a4/colfamily11/b6d28e100c1a43e8b426cbf04316f02c, entries=703, sequenceid=5, filesize=38.3 K 2024-11-12T14:52:39,103 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for bb0382e4edc08a7c43f0571deb2374a4 in 68ms, sequenceid=5, compaction requested=false 2024-11-12T14:52:39,112 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/d7c3d3ae3b942f0414dcbb110be8bc0b/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-12T14:52:39,113 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b. 2024-11-12T14:52:39,113 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d7c3d3ae3b942f0414dcbb110be8bc0b: Waiting for close lock at 1731423159090Running coprocessor pre-close hooks at 1731423159090Disabling compacts and flushes for region at 1731423159090Disabling writes for close at 1731423159091 (+1 ms)Writing region close event to WAL at 1731423159107 (+16 ms)Running coprocessor post-close hooks at 1731423159113 (+6 ms)Closed at 1731423159113 2024-11-12T14:52:39,113 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,nnn,1731422967229.d7c3d3ae3b942f0414dcbb110be8bc0b. 2024-11-12T14:52:39,114 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b419710b4e7b088b11da63dff9b7760e, disabling compactions & flushes 2024-11-12T14:52:39,114 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731422967229.b419710b4e7b088b11da63dff9b7760e. 2024-11-12T14:52:39,114 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731422967229.b419710b4e7b088b11da63dff9b7760e. 2024-11-12T14:52:39,114 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731422967229.b419710b4e7b088b11da63dff9b7760e. after waiting 0 ms 2024-11-12T14:52:39,114 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731422967229.b419710b4e7b088b11da63dff9b7760e. 2024-11-12T14:52:39,114 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing b419710b4e7b088b11da63dff9b7760e 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-12T14:52:39,116 INFO [regionserver/3945e1ca6f47:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-12T14:52:39,116 INFO [regionserver/3945e1ca6f47:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-12T14:52:39,118 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/bb0382e4edc08a7c43f0571deb2374a4/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-12T14:52:39,119 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731422967229.bb0382e4edc08a7c43f0571deb2374a4. 2024-11-12T14:52:39,119 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bb0382e4edc08a7c43f0571deb2374a4: Waiting for close lock at 1731423159035Running coprocessor pre-close hooks at 1731423159035Disabling compacts and flushes for region at 1731423159035Disabling writes for close at 1731423159035Obtaining lock to block concurrent updates at 1731423159036 (+1 ms)Preparing flush snapshotting stores in bb0382e4edc08a7c43f0571deb2374a4 at 1731423159036Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731422967229.bb0382e4edc08a7c43f0571deb2374a4., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731423159036Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731422967229.bb0382e4edc08a7c43f0571deb2374a4. at 1731423159037 (+1 ms)Flushing bb0382e4edc08a7c43f0571deb2374a4/colfamily11: creating writer at 1731423159037Flushing bb0382e4edc08a7c43f0571deb2374a4/colfamily11: appending metadata at 1731423159057 (+20 ms)Flushing bb0382e4edc08a7c43f0571deb2374a4/colfamily11: closing flushed file at 1731423159057Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55d96b7b: reopening flushed file at 1731423159092 (+35 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for bb0382e4edc08a7c43f0571deb2374a4 in 68ms, sequenceid=5, compaction requested=false at 1731423159103 (+11 ms)Writing region close event to WAL at 1731423159107 (+4 ms)Running coprocessor post-close hooks at 1731423159118 (+11 ms)Closed at 1731423159118 2024-11-12T14:52:39,119 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ooo,1731422967229.bb0382e4edc08a7c43f0571deb2374a4. 2024-11-12T14:52:39,119 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2b04a6d83bd2998d75a1743bdf61651e, disabling compactions & flushes 2024-11-12T14:52:39,119 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731422967229.2b04a6d83bd2998d75a1743bdf61651e. 2024-11-12T14:52:39,119 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731422967229.2b04a6d83bd2998d75a1743bdf61651e. 2024-11-12T14:52:39,119 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731422967229.2b04a6d83bd2998d75a1743bdf61651e. after waiting 0 ms 2024-11-12T14:52:39,119 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731422967229.2b04a6d83bd2998d75a1743bdf61651e. 2024-11-12T14:52:39,119 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 2b04a6d83bd2998d75a1743bdf61651e 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-12T14:52:39,126 DEBUG [RS:2;3945e1ca6f47:35473 {}] regionserver.HRegionServer(1351): Waiting on 2b04a6d83bd2998d75a1743bdf61651e, 8b5e68238ca126132dbf55f0fbf38047, 92f3e9baa717d57147319661f6f00e06, 9a6f7a226796c8ff7e21a40d463cd27b, f65e71cd750425cce9713e30a75c6002 2024-11-12T14:52:39,127 DEBUG [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(1351): Waiting on 06aa3b2e56ae4404b602d6fa98eabe33, 4936dfde0b7af7ea99d36c9e0c24fefc, b419710b4e7b088b11da63dff9b7760e, cf0006d9b8777107f458b7008fc42ba9, df409d6fa4df5d8fd91534d2d90eda60, e41036448a3677ed52f58f536690bf38, f943ce043844f379c1b7ecbf76b5a419 2024-11-12T14:52:39,132 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b419710b4e7b088b11da63dff9b7760e/.tmp/colfamily11/a35a5d9ce6c049a9a359a6a240d6ccad is 52, key is ppp/colfamily11:colfamily11/1731422971638/Put/seqid=0 2024-11-12T14:52:39,134 INFO [regionserver/3945e1ca6f47:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-12T14:52:39,134 INFO [regionserver/3945e1ca6f47:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-12T14:52:39,142 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2b04a6d83bd2998d75a1743bdf61651e/.tmp/colfamily11/5deca848d8cf4dcea9399fb25d2032a2 is 52, key is vvv/colfamily11:colfamily11/1731422971682/Put/seqid=0 2024-11-12T14:52:39,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741898_1074 (size=39263) 2024-11-12T14:52:39,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741898_1074 (size=39263) 2024-11-12T14:52:39,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741898_1074 (size=39263) 2024-11-12T14:52:39,149 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b419710b4e7b088b11da63dff9b7760e/.tmp/colfamily11/a35a5d9ce6c049a9a359a6a240d6ccad 2024-11-12T14:52:39,162 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b419710b4e7b088b11da63dff9b7760e/.tmp/colfamily11/a35a5d9ce6c049a9a359a6a240d6ccad as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b419710b4e7b088b11da63dff9b7760e/colfamily11/a35a5d9ce6c049a9a359a6a240d6ccad 2024-11-12T14:52:39,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741899_1075 (size=39263) 2024-11-12T14:52:39,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741899_1075 (size=39263) 2024-11-12T14:52:39,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741899_1075 (size=39263) 2024-11-12T14:52:39,179 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b419710b4e7b088b11da63dff9b7760e/colfamily11/a35a5d9ce6c049a9a359a6a240d6ccad, entries=703, sequenceid=5, filesize=38.3 K 2024-11-12T14:52:39,179 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2b04a6d83bd2998d75a1743bdf61651e/.tmp/colfamily11/5deca848d8cf4dcea9399fb25d2032a2 2024-11-12T14:52:39,180 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for b419710b4e7b088b11da63dff9b7760e in 66ms, sequenceid=5, compaction requested=false 2024-11-12T14:52:39,196 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2b04a6d83bd2998d75a1743bdf61651e/.tmp/colfamily11/5deca848d8cf4dcea9399fb25d2032a2 as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2b04a6d83bd2998d75a1743bdf61651e/colfamily11/5deca848d8cf4dcea9399fb25d2032a2 2024-11-12T14:52:39,204 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/b419710b4e7b088b11da63dff9b7760e/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-12T14:52:39,205 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731422967229.b419710b4e7b088b11da63dff9b7760e. 2024-11-12T14:52:39,205 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b419710b4e7b088b11da63dff9b7760e: Waiting for close lock at 1731423159113Running coprocessor pre-close hooks at 1731423159113Disabling compacts and flushes for region at 1731423159113Disabling writes for close at 1731423159114 (+1 ms)Obtaining lock to block concurrent updates at 1731423159114Preparing flush snapshotting stores in b419710b4e7b088b11da63dff9b7760e at 1731423159114Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731422967229.b419710b4e7b088b11da63dff9b7760e., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731423159114Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731422967229.b419710b4e7b088b11da63dff9b7760e. at 1731423159115 (+1 ms)Flushing b419710b4e7b088b11da63dff9b7760e/colfamily11: creating writer at 1731423159115Flushing b419710b4e7b088b11da63dff9b7760e/colfamily11: appending metadata at 1731423159132 (+17 ms)Flushing b419710b4e7b088b11da63dff9b7760e/colfamily11: closing flushed file at 1731423159132Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a4f8b08: reopening flushed file at 1731423159161 (+29 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for b419710b4e7b088b11da63dff9b7760e in 66ms, sequenceid=5, compaction requested=false at 1731423159180 (+19 ms)Writing region close event to WAL at 1731423159197 (+17 ms)Running coprocessor post-close hooks at 1731423159205 (+8 ms)Closed at 1731423159205 2024-11-12T14:52:39,206 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ppp,1731422967229.b419710b4e7b088b11da63dff9b7760e. 2024-11-12T14:52:39,206 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e41036448a3677ed52f58f536690bf38, disabling compactions & flushes 2024-11-12T14:52:39,206 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38. 2024-11-12T14:52:39,206 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38. 2024-11-12T14:52:39,206 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38. after waiting 0 ms 2024-11-12T14:52:39,206 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38. 2024-11-12T14:52:39,208 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2b04a6d83bd2998d75a1743bdf61651e/colfamily11/5deca848d8cf4dcea9399fb25d2032a2, entries=703, sequenceid=5, filesize=38.3 K 2024-11-12T14:52:39,209 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 2b04a6d83bd2998d75a1743bdf61651e in 90ms, sequenceid=5, compaction requested=false 2024-11-12T14:52:39,226 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/2b04a6d83bd2998d75a1743bdf61651e/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-12T14:52:39,226 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/e41036448a3677ed52f58f536690bf38/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-12T14:52:39,226 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731422967229.2b04a6d83bd2998d75a1743bdf61651e. 2024-11-12T14:52:39,226 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2b04a6d83bd2998d75a1743bdf61651e: Waiting for close lock at 1731423159119Running coprocessor pre-close hooks at 1731423159119Disabling compacts and flushes for region at 1731423159119Disabling writes for close at 1731423159119Obtaining lock to block concurrent updates at 1731423159119Preparing flush snapshotting stores in 2b04a6d83bd2998d75a1743bdf61651e at 1731423159119Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731422967229.2b04a6d83bd2998d75a1743bdf61651e., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731423159120 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731422967229.2b04a6d83bd2998d75a1743bdf61651e. at 1731423159120Flushing 2b04a6d83bd2998d75a1743bdf61651e/colfamily11: creating writer at 1731423159121 (+1 ms)Flushing 2b04a6d83bd2998d75a1743bdf61651e/colfamily11: appending metadata at 1731423159142 (+21 ms)Flushing 2b04a6d83bd2998d75a1743bdf61651e/colfamily11: closing flushed file at 1731423159142Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f98f563: reopening flushed file at 1731423159190 (+48 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 2b04a6d83bd2998d75a1743bdf61651e in 90ms, sequenceid=5, compaction requested=false at 1731423159209 (+19 ms)Writing region close event to WAL at 1731423159220 (+11 ms)Running coprocessor post-close hooks at 1731423159226 (+6 ms)Closed at 1731423159226 2024-11-12T14:52:39,226 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38. 2024-11-12T14:52:39,227 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e41036448a3677ed52f58f536690bf38: Waiting for close lock at 1731423159206Running coprocessor pre-close hooks at 1731423159206Disabling compacts and flushes for region at 1731423159206Disabling writes for close at 1731423159206Writing region close event to WAL at 1731423159219 (+13 ms)Running coprocessor post-close hooks at 1731423159226 (+7 ms)Closed at 1731423159226 2024-11-12T14:52:39,227 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,vvv,1731422967229.2b04a6d83bd2998d75a1743bdf61651e. 2024-11-12T14:52:39,227 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,zzz,1731422967229.e41036448a3677ed52f58f536690bf38. 2024-11-12T14:52:39,227 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 92f3e9baa717d57147319661f6f00e06, disabling compactions & flushes 2024-11-12T14:52:39,227 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f943ce043844f379c1b7ecbf76b5a419, disabling compactions & flushes 2024-11-12T14:52:39,227 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06. 2024-11-12T14:52:39,227 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419. 2024-11-12T14:52:39,227 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06. 2024-11-12T14:52:39,227 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419. 2024-11-12T14:52:39,227 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06. after waiting 0 ms 2024-11-12T14:52:39,227 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419. after waiting 0 ms 2024-11-12T14:52:39,227 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06. 2024-11-12T14:52:39,227 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419. 2024-11-12T14:52:39,227 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing f943ce043844f379c1b7ecbf76b5a419 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-12T14:52:39,235 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/92f3e9baa717d57147319661f6f00e06/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-12T14:52:39,237 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06. 2024-11-12T14:52:39,237 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 92f3e9baa717d57147319661f6f00e06: Waiting for close lock at 1731423159227Running coprocessor pre-close hooks at 1731423159227Disabling compacts and flushes for region at 1731423159227Disabling writes for close at 1731423159227Writing region close event to WAL at 1731423159228 (+1 ms)Running coprocessor post-close hooks at 1731423159237 (+9 ms)Closed at 1731423159237 2024-11-12T14:52:39,237 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,jjj,1731422967229.92f3e9baa717d57147319661f6f00e06. 2024-11-12T14:52:39,237 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8b5e68238ca126132dbf55f0fbf38047, disabling compactions & flushes 2024-11-12T14:52:39,237 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731422967229.8b5e68238ca126132dbf55f0fbf38047. 2024-11-12T14:52:39,237 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731422967229.8b5e68238ca126132dbf55f0fbf38047. 2024-11-12T14:52:39,238 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731422967229.8b5e68238ca126132dbf55f0fbf38047. after waiting 1 ms 2024-11-12T14:52:39,238 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731422967229.8b5e68238ca126132dbf55f0fbf38047. 2024-11-12T14:52:39,238 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 8b5e68238ca126132dbf55f0fbf38047 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-12T14:52:39,256 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f943ce043844f379c1b7ecbf76b5a419/.tmp/colfamily11/7d1f5e589a3740858ba8f3df835f6915 is 52, key is aaa/colfamily11:colfamily11/1731422971340/Put/seqid=0 2024-11-12T14:52:39,269 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8b5e68238ca126132dbf55f0fbf38047/.tmp/colfamily11/a2565dffc7324ceaa3db5939359deff7 is 52, key is sss/colfamily11:colfamily11/1731422971642/Put/seqid=0 2024-11-12T14:52:39,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741900_1076 (size=39263) 2024-11-12T14:52:39,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741900_1076 (size=39263) 2024-11-12T14:52:39,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741900_1076 (size=39263) 2024-11-12T14:52:39,276 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f943ce043844f379c1b7ecbf76b5a419/.tmp/colfamily11/7d1f5e589a3740858ba8f3df835f6915 2024-11-12T14:52:39,290 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f943ce043844f379c1b7ecbf76b5a419/.tmp/colfamily11/7d1f5e589a3740858ba8f3df835f6915 as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f943ce043844f379c1b7ecbf76b5a419/colfamily11/7d1f5e589a3740858ba8f3df835f6915 2024-11-12T14:52:39,298 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f943ce043844f379c1b7ecbf76b5a419/colfamily11/7d1f5e589a3740858ba8f3df835f6915, entries=703, sequenceid=5, filesize=38.3 K 2024-11-12T14:52:39,300 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for f943ce043844f379c1b7ecbf76b5a419 in 72ms, sequenceid=5, compaction requested=false 2024-11-12T14:52:39,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741901_1077 (size=39263) 2024-11-12T14:52:39,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741901_1077 (size=39263) 2024-11-12T14:52:39,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741901_1077 (size=39263) 2024-11-12T14:52:39,309 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8b5e68238ca126132dbf55f0fbf38047/.tmp/colfamily11/a2565dffc7324ceaa3db5939359deff7 2024-11-12T14:52:39,310 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f943ce043844f379c1b7ecbf76b5a419/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-12T14:52:39,311 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419. 2024-11-12T14:52:39,311 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f943ce043844f379c1b7ecbf76b5a419: Waiting for close lock at 1731423159227Running coprocessor pre-close hooks at 1731423159227Disabling compacts and flushes for region at 1731423159227Disabling writes for close at 1731423159227Obtaining lock to block concurrent updates at 1731423159227Preparing flush snapshotting stores in f943ce043844f379c1b7ecbf76b5a419 at 1731423159227Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731423159228 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419. at 1731423159228Flushing f943ce043844f379c1b7ecbf76b5a419/colfamily11: creating writer at 1731423159228Flushing f943ce043844f379c1b7ecbf76b5a419/colfamily11: appending metadata at 1731423159256 (+28 ms)Flushing f943ce043844f379c1b7ecbf76b5a419/colfamily11: closing flushed file at 1731423159256Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e7fd78a: reopening flushed file at 1731423159285 (+29 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for f943ce043844f379c1b7ecbf76b5a419 in 72ms, sequenceid=5, compaction requested=false at 1731423159300 (+15 ms)Writing region close event to WAL at 1731423159305 (+5 ms)Running coprocessor post-close hooks at 1731423159311 (+6 ms)Closed at 1731423159311 2024-11-12T14:52:39,311 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,,1731422967229.f943ce043844f379c1b7ecbf76b5a419. 2024-11-12T14:52:39,311 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing df409d6fa4df5d8fd91534d2d90eda60, disabling compactions & flushes 2024-11-12T14:52:39,311 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731422967229.df409d6fa4df5d8fd91534d2d90eda60. 2024-11-12T14:52:39,311 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731422967229.df409d6fa4df5d8fd91534d2d90eda60. 2024-11-12T14:52:39,311 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731422967229.df409d6fa4df5d8fd91534d2d90eda60. after waiting 0 ms 2024-11-12T14:52:39,311 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731422967229.df409d6fa4df5d8fd91534d2d90eda60. 2024-11-12T14:52:39,312 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing df409d6fa4df5d8fd91534d2d90eda60 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-12T14:52:39,319 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8b5e68238ca126132dbf55f0fbf38047/.tmp/colfamily11/a2565dffc7324ceaa3db5939359deff7 as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8b5e68238ca126132dbf55f0fbf38047/colfamily11/a2565dffc7324ceaa3db5939359deff7 2024-11-12T14:52:39,326 DEBUG [RS:2;3945e1ca6f47:35473 {}] regionserver.HRegionServer(1351): Waiting on 8b5e68238ca126132dbf55f0fbf38047, 9a6f7a226796c8ff7e21a40d463cd27b, f65e71cd750425cce9713e30a75c6002 2024-11-12T14:52:39,327 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8b5e68238ca126132dbf55f0fbf38047/colfamily11/a2565dffc7324ceaa3db5939359deff7, entries=703, sequenceid=5, filesize=38.3 K 2024-11-12T14:52:39,328 DEBUG [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(1351): Waiting on 06aa3b2e56ae4404b602d6fa98eabe33, 4936dfde0b7af7ea99d36c9e0c24fefc, cf0006d9b8777107f458b7008fc42ba9, df409d6fa4df5d8fd91534d2d90eda60 2024-11-12T14:52:39,328 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 8b5e68238ca126132dbf55f0fbf38047 in 90ms, sequenceid=5, compaction requested=false 2024-11-12T14:52:39,333 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/df409d6fa4df5d8fd91534d2d90eda60/.tmp/colfamily11/3f937c82483e4d4da1c95cba36611ebc is 52, key is iii/colfamily11:colfamily11/1731422971487/Put/seqid=0 2024-11-12T14:52:39,333 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/8b5e68238ca126132dbf55f0fbf38047/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-12T14:52:39,334 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731422967229.8b5e68238ca126132dbf55f0fbf38047. 2024-11-12T14:52:39,334 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8b5e68238ca126132dbf55f0fbf38047: Waiting for close lock at 1731423159237Running coprocessor pre-close hooks at 1731423159237Disabling compacts and flushes for region at 1731423159237Disabling writes for close at 1731423159238 (+1 ms)Obtaining lock to block concurrent updates at 1731423159238Preparing flush snapshotting stores in 8b5e68238ca126132dbf55f0fbf38047 at 1731423159238Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731422967229.8b5e68238ca126132dbf55f0fbf38047., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731423159239 (+1 ms)Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731422967229.8b5e68238ca126132dbf55f0fbf38047. at 1731423159245 (+6 ms)Flushing 8b5e68238ca126132dbf55f0fbf38047/colfamily11: creating writer at 1731423159245Flushing 8b5e68238ca126132dbf55f0fbf38047/colfamily11: appending metadata at 1731423159269 (+24 ms)Flushing 8b5e68238ca126132dbf55f0fbf38047/colfamily11: closing flushed file at 1731423159269Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35bfa781: reopening flushed file at 1731423159318 (+49 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for 8b5e68238ca126132dbf55f0fbf38047 in 90ms, sequenceid=5, compaction requested=false at 1731423159328 (+10 ms)Writing region close event to WAL at 1731423159329 (+1 ms)Running coprocessor post-close hooks at 1731423159334 (+5 ms)Closed at 1731423159334 2024-11-12T14:52:39,334 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,sss,1731422967229.8b5e68238ca126132dbf55f0fbf38047. 2024-11-12T14:52:39,335 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9a6f7a226796c8ff7e21a40d463cd27b, disabling compactions & flushes 2024-11-12T14:52:39,335 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b. 2024-11-12T14:52:39,335 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b. 2024-11-12T14:52:39,335 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b. after waiting 0 ms 2024-11-12T14:52:39,335 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b. 2024-11-12T14:52:39,346 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/9a6f7a226796c8ff7e21a40d463cd27b/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-12T14:52:39,348 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b. 2024-11-12T14:52:39,348 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9a6f7a226796c8ff7e21a40d463cd27b: Waiting for close lock at 1731423159335Running coprocessor pre-close hooks at 1731423159335Disabling compacts and flushes for region at 1731423159335Disabling writes for close at 1731423159335Writing region close event to WAL at 1731423159337 (+2 ms)Running coprocessor post-close hooks at 1731423159347 (+10 ms)Closed at 1731423159348 (+1 ms) 2024-11-12T14:52:39,348 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,www,1731422967229.9a6f7a226796c8ff7e21a40d463cd27b. 2024-11-12T14:52:39,348 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f65e71cd750425cce9713e30a75c6002, disabling compactions & flushes 2024-11-12T14:52:39,348 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731422967229.f65e71cd750425cce9713e30a75c6002. 2024-11-12T14:52:39,348 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731422967229.f65e71cd750425cce9713e30a75c6002. 2024-11-12T14:52:39,348 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731422967229.f65e71cd750425cce9713e30a75c6002. after waiting 0 ms 2024-11-12T14:52:39,348 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731422967229.f65e71cd750425cce9713e30a75c6002. 2024-11-12T14:52:39,349 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing f65e71cd750425cce9713e30a75c6002 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-12T14:52:39,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741902_1078 (size=39263) 2024-11-12T14:52:39,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741902_1078 (size=39263) 2024-11-12T14:52:39,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741902_1078 (size=39263) 2024-11-12T14:52:39,367 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/df409d6fa4df5d8fd91534d2d90eda60/.tmp/colfamily11/3f937c82483e4d4da1c95cba36611ebc 2024-11-12T14:52:39,377 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f65e71cd750425cce9713e30a75c6002/.tmp/colfamily11/ae048edcbafb4960af97b6d221741d75 is 52, key is hhh/colfamily11:colfamily11/1731422971516/Put/seqid=0 2024-11-12T14:52:39,378 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/df409d6fa4df5d8fd91534d2d90eda60/.tmp/colfamily11/3f937c82483e4d4da1c95cba36611ebc as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/df409d6fa4df5d8fd91534d2d90eda60/colfamily11/3f937c82483e4d4da1c95cba36611ebc 2024-11-12T14:52:39,387 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/df409d6fa4df5d8fd91534d2d90eda60/colfamily11/3f937c82483e4d4da1c95cba36611ebc, entries=703, sequenceid=5, filesize=38.3 K 2024-11-12T14:52:39,389 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for df409d6fa4df5d8fd91534d2d90eda60 in 77ms, sequenceid=5, compaction requested=false 2024-11-12T14:52:39,402 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/df409d6fa4df5d8fd91534d2d90eda60/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-12T14:52:39,403 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731422967229.df409d6fa4df5d8fd91534d2d90eda60. 2024-11-12T14:52:39,403 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for df409d6fa4df5d8fd91534d2d90eda60: Waiting for close lock at 1731423159311Running coprocessor pre-close hooks at 1731423159311Disabling compacts and flushes for region at 1731423159311Disabling writes for close at 1731423159311Obtaining lock to block concurrent updates at 1731423159312 (+1 ms)Preparing flush snapshotting stores in df409d6fa4df5d8fd91534d2d90eda60 at 1731423159312Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731422967229.df409d6fa4df5d8fd91534d2d90eda60., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731423159312Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731422967229.df409d6fa4df5d8fd91534d2d90eda60. at 1731423159313 (+1 ms)Flushing df409d6fa4df5d8fd91534d2d90eda60/colfamily11: creating writer at 1731423159313Flushing df409d6fa4df5d8fd91534d2d90eda60/colfamily11: appending metadata at 1731423159333 (+20 ms)Flushing df409d6fa4df5d8fd91534d2d90eda60/colfamily11: closing flushed file at 1731423159333Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26e7a9d0: reopening flushed file at 1731423159376 (+43 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for df409d6fa4df5d8fd91534d2d90eda60 in 77ms, sequenceid=5, compaction requested=false at 1731423159389 (+13 ms)Writing region close event to WAL at 1731423159398 (+9 ms)Running coprocessor post-close hooks at 1731423159403 (+5 ms)Closed at 1731423159403 2024-11-12T14:52:39,403 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,iii,1731422967229.df409d6fa4df5d8fd91534d2d90eda60. 2024-11-12T14:52:39,403 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4936dfde0b7af7ea99d36c9e0c24fefc, disabling compactions & flushes 2024-11-12T14:52:39,403 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc. 2024-11-12T14:52:39,403 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc. 2024-11-12T14:52:39,403 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc. after waiting 0 ms 2024-11-12T14:52:39,403 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc. 2024-11-12T14:52:39,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741903_1079 (size=39263) 2024-11-12T14:52:39,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741903_1079 (size=39263) 2024-11-12T14:52:39,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741903_1079 (size=39263) 2024-11-12T14:52:39,417 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f65e71cd750425cce9713e30a75c6002/.tmp/colfamily11/ae048edcbafb4960af97b6d221741d75 2024-11-12T14:52:39,422 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/4936dfde0b7af7ea99d36c9e0c24fefc/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-12T14:52:39,423 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc. 2024-11-12T14:52:39,423 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4936dfde0b7af7ea99d36c9e0c24fefc: Waiting for close lock at 1731423159403Running coprocessor pre-close hooks at 1731423159403Disabling compacts and flushes for region at 1731423159403Disabling writes for close at 1731423159403Writing region close event to WAL at 1731423159413 (+10 ms)Running coprocessor post-close hooks at 1731423159423 (+10 ms)Closed at 1731423159423 2024-11-12T14:52:39,423 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ggg,1731422967229.4936dfde0b7af7ea99d36c9e0c24fefc. 2024-11-12T14:52:39,423 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 06aa3b2e56ae4404b602d6fa98eabe33, disabling compactions & flushes 2024-11-12T14:52:39,423 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33. 2024-11-12T14:52:39,423 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33. 2024-11-12T14:52:39,424 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33. after waiting 0 ms 2024-11-12T14:52:39,424 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33. 2024-11-12T14:52:39,429 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/06aa3b2e56ae4404b602d6fa98eabe33/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-12T14:52:39,430 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33. 2024-11-12T14:52:39,430 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 06aa3b2e56ae4404b602d6fa98eabe33: Waiting for close lock at 1731423159423Running coprocessor pre-close hooks at 1731423159423Disabling compacts and flushes for region at 1731423159423Disabling writes for close at 1731423159424 (+1 ms)Writing region close event to WAL at 1731423159425 (+1 ms)Running coprocessor post-close hooks at 1731423159430 (+5 ms)Closed at 1731423159430 2024-11-12T14:52:39,430 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ccc,1731422967229.06aa3b2e56ae4404b602d6fa98eabe33. 2024-11-12T14:52:39,430 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cf0006d9b8777107f458b7008fc42ba9, disabling compactions & flushes 2024-11-12T14:52:39,430 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731422967229.cf0006d9b8777107f458b7008fc42ba9. 2024-11-12T14:52:39,430 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731422967229.cf0006d9b8777107f458b7008fc42ba9. 2024-11-12T14:52:39,431 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731422967229.cf0006d9b8777107f458b7008fc42ba9. after waiting 0 ms 2024-11-12T14:52:39,431 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731422967229.cf0006d9b8777107f458b7008fc42ba9. 2024-11-12T14:52:39,431 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing cf0006d9b8777107f458b7008fc42ba9 3/3 column families, dataSize=32.95 KB heapSize=83.13 KB 2024-11-12T14:52:39,432 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f65e71cd750425cce9713e30a75c6002/.tmp/colfamily11/ae048edcbafb4960af97b6d221741d75 as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f65e71cd750425cce9713e30a75c6002/colfamily11/ae048edcbafb4960af97b6d221741d75 2024-11-12T14:52:39,444 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f65e71cd750425cce9713e30a75c6002/colfamily11/ae048edcbafb4960af97b6d221741d75, entries=703, sequenceid=5, filesize=38.3 K 2024-11-12T14:52:39,446 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for f65e71cd750425cce9713e30a75c6002 in 98ms, sequenceid=5, compaction requested=false 2024-11-12T14:52:39,454 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/f65e71cd750425cce9713e30a75c6002/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-12T14:52:39,455 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731422967229.f65e71cd750425cce9713e30a75c6002. 2024-11-12T14:52:39,455 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f65e71cd750425cce9713e30a75c6002: Waiting for close lock at 1731423159348Running coprocessor pre-close hooks at 1731423159348Disabling compacts and flushes for region at 1731423159348Disabling writes for close at 1731423159348Obtaining lock to block concurrent updates at 1731423159349 (+1 ms)Preparing flush snapshotting stores in f65e71cd750425cce9713e30a75c6002 at 1731423159349Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731422967229.f65e71cd750425cce9713e30a75c6002., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731423159349Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731422967229.f65e71cd750425cce9713e30a75c6002. at 1731423159350 (+1 ms)Flushing f65e71cd750425cce9713e30a75c6002/colfamily11: creating writer at 1731423159350Flushing f65e71cd750425cce9713e30a75c6002/colfamily11: appending metadata at 1731423159376 (+26 ms)Flushing f65e71cd750425cce9713e30a75c6002/colfamily11: closing flushed file at 1731423159377 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@276a16e0: reopening flushed file at 1731423159431 (+54 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for f65e71cd750425cce9713e30a75c6002 in 98ms, sequenceid=5, compaction requested=false at 1731423159446 (+15 ms)Writing region close event to WAL at 1731423159449 (+3 ms)Running coprocessor post-close hooks at 1731423159455 (+6 ms)Closed at 1731423159455 2024-11-12T14:52:39,455 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,hhh,1731422967229.f65e71cd750425cce9713e30a75c6002. 2024-11-12T14:52:39,461 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/cf0006d9b8777107f458b7008fc42ba9/.tmp/colfamily11/7f45e1846a8f43bcb6f6d3a0375a2ea2 is 52, key is ddd/colfamily11:colfamily11/1731422971431/Put/seqid=0 2024-11-12T14:52:39,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741904_1080 (size=39263) 2024-11-12T14:52:39,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741904_1080 (size=39263) 2024-11-12T14:52:39,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741904_1080 (size=39263) 2024-11-12T14:52:39,472 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.95 KB at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/cf0006d9b8777107f458b7008fc42ba9/.tmp/colfamily11/7f45e1846a8f43bcb6f6d3a0375a2ea2 2024-11-12T14:52:39,479 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/cf0006d9b8777107f458b7008fc42ba9/.tmp/colfamily11/7f45e1846a8f43bcb6f6d3a0375a2ea2 as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/cf0006d9b8777107f458b7008fc42ba9/colfamily11/7f45e1846a8f43bcb6f6d3a0375a2ea2 2024-11-12T14:52:39,487 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/cf0006d9b8777107f458b7008fc42ba9/colfamily11/7f45e1846a8f43bcb6f6d3a0375a2ea2, entries=703, sequenceid=5, filesize=38.3 K 2024-11-12T14:52:39,488 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for cf0006d9b8777107f458b7008fc42ba9 in 57ms, sequenceid=5, compaction requested=false 2024-11-12T14:52:39,493 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/data/default/testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true/cf0006d9b8777107f458b7008fc42ba9/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-12T14:52:39,494 INFO [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731422967229.cf0006d9b8777107f458b7008fc42ba9. 2024-11-12T14:52:39,494 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cf0006d9b8777107f458b7008fc42ba9: Waiting for close lock at 1731423159430Running coprocessor pre-close hooks at 1731423159430Disabling compacts and flushes for region at 1731423159430Disabling writes for close at 1731423159431 (+1 ms)Obtaining lock to block concurrent updates at 1731423159431Preparing flush snapshotting stores in cf0006d9b8777107f458b7008fc42ba9 at 1731423159431Finished memstore snapshotting testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731422967229.cf0006d9b8777107f458b7008fc42ba9., syncing WAL and waiting on mvcc, flushsize=dataSize=33744, getHeapSize=85080, getOffHeapSize=0, getCellsCount=703 at 1731423159431Flushing stores of testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731422967229.cf0006d9b8777107f458b7008fc42ba9. at 1731423159432 (+1 ms)Flushing cf0006d9b8777107f458b7008fc42ba9/colfamily11: creating writer at 1731423159432Flushing cf0006d9b8777107f458b7008fc42ba9/colfamily11: appending metadata at 1731423159461 (+29 ms)Flushing cf0006d9b8777107f458b7008fc42ba9/colfamily11: closing flushed file at 1731423159461Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d19437: reopening flushed file at 1731423159478 (+17 ms)Finished flush of dataSize ~32.95 KB/33744, heapSize ~82.62 KB/84600, currentSize=0 B/0 for cf0006d9b8777107f458b7008fc42ba9 in 57ms, sequenceid=5, compaction requested=false at 1731423159488 (+10 ms)Writing region close event to WAL at 1731423159489 (+1 ms)Running coprocessor post-close hooks at 1731423159494 (+5 ms)Closed at 1731423159494 2024-11-12T14:52:39,495 DEBUG [RS_CLOSE_REGION-regionserver/3945e1ca6f47:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRecoveryAndDoubleExecution-carryingMeta-true-doubleExecution-true,ddd,1731422967229.cf0006d9b8777107f458b7008fc42ba9. 2024-11-12T14:52:39,526 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.HRegionServer(976): stopping server 3945e1ca6f47,35473,1731422964757; all regions closed. 2024-11-12T14:52:39,528 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(976): stopping server 3945e1ca6f47,45845,1731422964694; all regions closed. 2024-11-12T14:52:39,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741834_1010 (size=314313) 2024-11-12T14:52:39,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741834_1010 (size=314313) 2024-11-12T14:52:39,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741834_1010 (size=314313) 2024-11-12T14:52:39,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741865_1041 (size=15746) 2024-11-12T14:52:39,534 DEBUG [RS:2;3945e1ca6f47:35473 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/oldWALs 2024-11-12T14:52:39,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741865_1041 (size=15746) 2024-11-12T14:52:39,534 INFO [RS:2;3945e1ca6f47:35473 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3945e1ca6f47%2C35473%2C1731422964757:(num 1731422966318) 2024-11-12T14:52:39,534 DEBUG [RS:2;3945e1ca6f47:35473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:52:39,534 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T14:52:39,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741865_1041 (size=15746) 2024-11-12T14:52:39,534 INFO [RS:2;3945e1ca6f47:35473 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T14:52:39,535 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/WALs/3945e1ca6f47,45845,1731422964694/3945e1ca6f47%2C45845%2C1731422964694.meta.1731423153040.meta not finished, retry = 0 2024-11-12T14:52:39,535 INFO [RS:2;3945e1ca6f47:35473 {}] hbase.ChoreService(370): Chore service for: regionserver/3945e1ca6f47:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T14:52:39,535 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T14:52:39,535 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T14:52:39,535 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T14:52:39,535 INFO [regionserver/3945e1ca6f47:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T14:52:39,535 INFO [RS:2;3945e1ca6f47:35473 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T14:52:39,535 INFO [RS:2;3945e1ca6f47:35473 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35473 2024-11-12T14:52:39,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T14:52:39,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3945e1ca6f47,35473,1731422964757 2024-11-12T14:52:39,538 INFO [RS:2;3945e1ca6f47:35473 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T14:52:39,539 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3945e1ca6f47,35473,1731422964757] 2024-11-12T14:52:39,540 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3945e1ca6f47,35473,1731422964757 already deleted, retry=false 2024-11-12T14:52:39,541 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 3945e1ca6f47,35473,1731422964757 expired; onlineServers=1 2024-11-12T14:52:39,638 DEBUG [RS:1;3945e1ca6f47:45845 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/oldWALs 2024-11-12T14:52:39,638 INFO [RS:1;3945e1ca6f47:45845 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3945e1ca6f47%2C45845%2C1731422964694.meta:.meta(num 1731423153040) 2024-11-12T14:52:39,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:52:39,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35473-0x1003469214f0003, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:52:39,639 INFO [RS:2;3945e1ca6f47:35473 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T14:52:39,639 INFO [RS:2;3945e1ca6f47:35473 {}] regionserver.HRegionServer(1031): Exiting; stopping=3945e1ca6f47,35473,1731422964757; zookeeper connection closed. 2024-11-12T14:52:39,640 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4187cde5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4187cde5 2024-11-12T14:52:39,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741835_1011 (size=354117) 2024-11-12T14:52:39,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741835_1011 (size=354117) 2024-11-12T14:52:39,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741835_1011 (size=354117) 2024-11-12T14:52:39,644 DEBUG [RS:1;3945e1ca6f47:45845 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/oldWALs 2024-11-12T14:52:39,644 INFO [RS:1;3945e1ca6f47:45845 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 3945e1ca6f47%2C45845%2C1731422964694:(num 1731422966317) 2024-11-12T14:52:39,644 DEBUG [RS:1;3945e1ca6f47:45845 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:52:39,644 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T14:52:39,644 INFO [RS:1;3945e1ca6f47:45845 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T14:52:39,645 INFO [RS:1;3945e1ca6f47:45845 {}] hbase.ChoreService(370): Chore service for: regionserver/3945e1ca6f47:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-12T14:52:39,645 INFO [RS:1;3945e1ca6f47:45845 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T14:52:39,645 INFO [regionserver/3945e1ca6f47:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T14:52:39,645 INFO [RS:1;3945e1ca6f47:45845 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45845 2024-11-12T14:52:39,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/3945e1ca6f47,45845,1731422964694 2024-11-12T14:52:39,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T14:52:39,646 INFO [RS:1;3945e1ca6f47:45845 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T14:52:39,647 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [3945e1ca6f47,45845,1731422964694] 2024-11-12T14:52:39,648 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/3945e1ca6f47,45845,1731422964694 already deleted, retry=false 2024-11-12T14:52:39,648 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 3945e1ca6f47,45845,1731422964694 expired; onlineServers=0 2024-11-12T14:52:39,648 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '3945e1ca6f47,38303,1731422963767' ***** 2024-11-12T14:52:39,648 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-12T14:52:39,648 INFO [M:0;3945e1ca6f47:38303 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T14:52:39,648 INFO [M:0;3945e1ca6f47:38303 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T14:52:39,648 DEBUG [M:0;3945e1ca6f47:38303 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-12T14:52:39,648 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-12T14:52:39,648 DEBUG [M:0;3945e1ca6f47:38303 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-12T14:52:39,648 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster-HFileCleaner.small.0-1731422965871 {}] cleaner.HFileCleaner(306): Exit Thread[master/3945e1ca6f47:0:becomeActiveMaster-HFileCleaner.small.0-1731422965871,5,FailOnTimeoutGroup] 2024-11-12T14:52:39,648 DEBUG [master/3945e1ca6f47:0:becomeActiveMaster-HFileCleaner.large.0-1731422965867 {}] cleaner.HFileCleaner(306): Exit Thread[master/3945e1ca6f47:0:becomeActiveMaster-HFileCleaner.large.0-1731422965867,5,FailOnTimeoutGroup] 2024-11-12T14:52:39,649 INFO [M:0;3945e1ca6f47:38303 {}] hbase.ChoreService(370): Chore service for: master/3945e1ca6f47:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-12T14:52:39,649 INFO [M:0;3945e1ca6f47:38303 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T14:52:39,649 DEBUG [M:0;3945e1ca6f47:38303 {}] master.HMaster(1795): Stopping service threads 2024-11-12T14:52:39,649 INFO [M:0;3945e1ca6f47:38303 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-12T14:52:39,649 INFO [M:0;3945e1ca6f47:38303 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T14:52:39,649 INFO [M:0;3945e1ca6f47:38303 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T14:52:39,649 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-12T14:52:39,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-12T14:52:39,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:52:39,650 DEBUG [M:0;3945e1ca6f47:38303 {}] zookeeper.ZKUtil(347): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-12T14:52:39,650 WARN [M:0;3945e1ca6f47:38303 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-12T14:52:39,650 INFO [M:0;3945e1ca6f47:38303 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/.lastflushedseqids 2024-11-12T14:52:39,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741905_1081 (size=2837) 2024-11-12T14:52:39,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741905_1081 (size=2837) 2024-11-12T14:52:39,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741905_1081 (size=2837) 2024-11-12T14:52:39,663 INFO [M:0;3945e1ca6f47:38303 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T14:52:39,663 INFO [M:0;3945e1ca6f47:38303 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-12T14:52:39,663 DEBUG [M:0;3945e1ca6f47:38303 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T14:52:39,663 INFO [M:0;3945e1ca6f47:38303 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:52:39,663 DEBUG [M:0;3945e1ca6f47:38303 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:52:39,663 DEBUG [M:0;3945e1ca6f47:38303 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T14:52:39,664 DEBUG [M:0;3945e1ca6f47:38303 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:52:39,664 INFO [M:0;3945e1ca6f47:38303 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=344.57 KB heapSize=404.97 KB 2024-11-12T14:52:39,687 DEBUG [M:0;3945e1ca6f47:38303 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3c7297e6ccfa47909e3dbda81ed67db5 is 82, key is hbase:meta,,1/info:regioninfo/1731423153499/Put/seqid=0 2024-11-12T14:52:39,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741906_1082 (size=6071) 2024-11-12T14:52:39,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741906_1082 (size=6071) 2024-11-12T14:52:39,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741906_1082 (size=6071) 2024-11-12T14:52:39,695 INFO [M:0;3945e1ca6f47:38303 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3c7297e6ccfa47909e3dbda81ed67db5 2024-11-12T14:52:39,715 DEBUG [M:0;3945e1ca6f47:38303 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c56d66a400cf4f218e6058dd2ae5322a is 4106, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731422969168/Put/seqid=0 2024-11-12T14:52:39,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741907_1083 (size=13503) 2024-11-12T14:52:39,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741907_1083 (size=13503) 2024-11-12T14:52:39,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741907_1083 (size=13503) 2024-11-12T14:52:39,725 INFO [M:0;3945e1ca6f47:38303 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=343.35 KB at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c56d66a400cf4f218e6058dd2ae5322a 2024-11-12T14:52:39,747 DEBUG [M:0;3945e1ca6f47:38303 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b306869eaa3f4500a9715e568b3bbee7 is 69, key is 3945e1ca6f47,35473,1731422964757/rs:state/1731422965969/Put/seqid=0 2024-11-12T14:52:39,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:52:39,747 INFO [RS:1;3945e1ca6f47:45845 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T14:52:39,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45845-0x1003469214f0002, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:52:39,747 INFO [RS:1;3945e1ca6f47:45845 {}] regionserver.HRegionServer(1031): Exiting; stopping=3945e1ca6f47,45845,1731422964694; zookeeper connection closed. 2024-11-12T14:52:39,748 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@79a0430 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@79a0430 2024-11-12T14:52:39,748 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-12T14:52:39,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741908_1084 (size=5447) 2024-11-12T14:52:39,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741908_1084 (size=5447) 2024-11-12T14:52:39,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741908_1084 (size=5447) 2024-11-12T14:52:39,765 INFO [M:0;3945e1ca6f47:38303 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b306869eaa3f4500a9715e568b3bbee7 2024-11-12T14:52:39,773 INFO [M:0;3945e1ca6f47:38303 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b306869eaa3f4500a9715e568b3bbee7 2024-11-12T14:52:39,774 DEBUG [M:0;3945e1ca6f47:38303 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3c7297e6ccfa47909e3dbda81ed67db5 as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3c7297e6ccfa47909e3dbda81ed67db5 2024-11-12T14:52:39,782 INFO [M:0;3945e1ca6f47:38303 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3c7297e6ccfa47909e3dbda81ed67db5, entries=14, sequenceid=635, filesize=5.9 K 2024-11-12T14:52:39,783 DEBUG [M:0;3945e1ca6f47:38303 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c56d66a400cf4f218e6058dd2ae5322a as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c56d66a400cf4f218e6058dd2ae5322a 2024-11-12T14:52:39,790 INFO [M:0;3945e1ca6f47:38303 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c56d66a400cf4f218e6058dd2ae5322a, entries=81, sequenceid=635, filesize=13.2 K 2024-11-12T14:52:39,792 DEBUG [M:0;3945e1ca6f47:38303 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b306869eaa3f4500a9715e568b3bbee7 as hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b306869eaa3f4500a9715e568b3bbee7 2024-11-12T14:52:39,799 INFO [M:0;3945e1ca6f47:38303 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b306869eaa3f4500a9715e568b3bbee7 2024-11-12T14:52:39,799 INFO [M:0;3945e1ca6f47:38303 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37295/user/jenkins/test-data/493ad17d-f368-1ba2-9d00-131773489050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b306869eaa3f4500a9715e568b3bbee7, entries=3, sequenceid=635, filesize=5.3 K 2024-11-12T14:52:39,801 INFO [M:0;3945e1ca6f47:38303 {}] regionserver.HRegion(3140): Finished flush of dataSize ~344.57 KB/352844, heapSize ~404.67 KB/414384, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 136ms, sequenceid=635, compaction requested=false 2024-11-12T14:52:39,813 INFO [M:0;3945e1ca6f47:38303 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:52:39,813 DEBUG [M:0;3945e1ca6f47:38303 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731423159663Disabling compacts and flushes for region at 1731423159663Disabling writes for close at 1731423159663Obtaining lock to block concurrent updates at 1731423159664 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731423159664Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=352844, getHeapSize=414624, getOffHeapSize=0, getCellsCount=800 at 1731423159664Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731423159664Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731423159665 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731423159687 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731423159687Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731423159700 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731423159715 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731423159715Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731423159732 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731423159746 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731423159747 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2be3f256: reopening flushed file at 1731423159773 (+26 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@622d7013: reopening flushed file at 1731423159782 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34bef6b8: reopening flushed file at 1731423159791 (+9 ms)Finished flush of dataSize ~344.57 KB/352844, heapSize ~404.67 KB/414384, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 136ms, sequenceid=635, compaction requested=false at 1731423159801 (+10 ms)Writing region close event to WAL at 1731423159813 (+12 ms)Closed at 1731423159813 2024-11-12T14:52:39,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41429 is added to blk_1073741830_1006 (size=395198) 2024-11-12T14:52:39,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44401 is added to blk_1073741830_1006 (size=395198) 2024-11-12T14:52:39,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38947 is added to blk_1073741830_1006 (size=395198) 2024-11-12T14:52:39,825 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T14:52:39,825 INFO [M:0;3945e1ca6f47:38303 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-12T14:52:39,825 INFO [M:0;3945e1ca6f47:38303 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38303 2024-11-12T14:52:39,825 INFO [M:0;3945e1ca6f47:38303 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T14:52:39,927 INFO [M:0;3945e1ca6f47:38303 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T14:52:39,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:52:39,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38303-0x1003469214f0000, quorum=127.0.0.1:62355, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:52:39,931 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@486c2677{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T14:52:39,934 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6738d157{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T14:52:39,934 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T14:52:39,934 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@43bf9dc1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T14:52:39,934 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ed2891b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/hadoop.log.dir/,STOPPED} 2024-11-12T14:52:39,937 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-12T14:52:39,937 WARN [BP-619972518-172.17.0.2-1731422960904 heartbeating to localhost/127.0.0.1:37295 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-12T14:52:39,937 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-12T14:52:39,937 WARN [BP-619972518-172.17.0.2-1731422960904 heartbeating to localhost/127.0.0.1:37295 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-619972518-172.17.0.2-1731422960904 (Datanode Uuid db6abbf5-482f-46a9-8114-3d0e0f8fbd8a) service to localhost/127.0.0.1:37295 2024-11-12T14:52:39,939 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/cluster_fa7c45b8-f5f8-b08a-b595-b8fe56346db2/data/data5/current/BP-619972518-172.17.0.2-1731422960904 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T14:52:39,939 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/cluster_fa7c45b8-f5f8-b08a-b595-b8fe56346db2/data/data6/current/BP-619972518-172.17.0.2-1731422960904 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T14:52:39,939 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-12T14:52:39,941 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3635c31e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T14:52:39,941 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@78839a64{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T14:52:39,941 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T14:52:39,942 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a292a59{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T14:52:39,942 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@63779849{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/hadoop.log.dir/,STOPPED} 2024-11-12T14:52:39,943 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-12T14:52:39,943 WARN [BP-619972518-172.17.0.2-1731422960904 heartbeating to localhost/127.0.0.1:37295 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-12T14:52:39,943 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-12T14:52:39,943 WARN [BP-619972518-172.17.0.2-1731422960904 heartbeating to localhost/127.0.0.1:37295 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-619972518-172.17.0.2-1731422960904 (Datanode Uuid d624de6c-b131-493c-901a-01994b4c38f0) service to localhost/127.0.0.1:37295 2024-11-12T14:52:39,943 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/cluster_fa7c45b8-f5f8-b08a-b595-b8fe56346db2/data/data3/current/BP-619972518-172.17.0.2-1731422960904 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T14:52:39,944 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/cluster_fa7c45b8-f5f8-b08a-b595-b8fe56346db2/data/data4/current/BP-619972518-172.17.0.2-1731422960904 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T14:52:39,944 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-12T14:52:39,945 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4255463d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T14:52:39,946 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5e48d68a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T14:52:39,946 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T14:52:39,946 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@63d82da9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T14:52:39,946 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b71ffd0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/hadoop.log.dir/,STOPPED} 2024-11-12T14:52:39,947 WARN [BP-619972518-172.17.0.2-1731422960904 heartbeating to localhost/127.0.0.1:37295 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-12T14:52:39,947 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-12T14:52:39,947 WARN [BP-619972518-172.17.0.2-1731422960904 heartbeating to localhost/127.0.0.1:37295 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-619972518-172.17.0.2-1731422960904 (Datanode Uuid 087d5b94-7f1d-46ba-8026-5884f8bf8180) service to localhost/127.0.0.1:37295 2024-11-12T14:52:39,947 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-12T14:52:39,948 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/cluster_fa7c45b8-f5f8-b08a-b595-b8fe56346db2/data/data1/current/BP-619972518-172.17.0.2-1731422960904 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T14:52:39,948 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/cluster_fa7c45b8-f5f8-b08a-b595-b8fe56346db2/data/data2/current/BP-619972518-172.17.0.2-1731422960904 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T14:52:39,948 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-12T14:52:39,956 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1aa62678{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T14:52:39,957 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1c0c42ad{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T14:52:39,957 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T14:52:39,958 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2a372edd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T14:52:39,958 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@58897554{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f95619f1-ff98-3cf0-9ad6-f516eec04b2c/hadoop.log.dir/,STOPPED} 2024-11-12T14:52:39,972 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-12T14:52:40,049 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-12T14:52:40,060 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.procedure.TestSCPWithMeta#testRecoveryAndDoubleExecutionOnRsWithMeta Thread=89 (was 11) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37295 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SSL Certificates Store Monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37295 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/3945e1ca6f47:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/3945e1ca6f47:0.logRoller java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:180) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37295 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@1c4da849 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: BootstrapNodeManager java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_META-regionserver/3945e1ca6f47:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/3945e1ca6f47:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-6-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/3945e1ca6f47:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: nioEventLoopGroup-7-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (608921412) connection to localhost/127.0.0.1:37295 from jenkins.hfs.1 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-7-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (608921412) connection to localhost/127.0.0.1:37295 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:37295 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/3945e1ca6f47:0.Chore.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/3945e1ca6f47:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-6-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/3945e1ca6f47:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:37295 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:37295 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37295 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: nioEventLoopGroup-4-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-6-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/3945e1ca6f47:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (608921412) connection to localhost/127.0.0.1:37295 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (608921412) connection to localhost/127.0.0.1:37295 from jenkins.hfs.2 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-2-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Abort regionserver monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-7-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 285) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=154 (was 149) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7833 (was 8848)