2024-11-13 07:13:15,895 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7a6d7e92 2024-11-13 07:13:15,921 main DEBUG Took 0.020624 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 07:13:15,922 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 07:13:15,922 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 07:13:15,924 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 07:13:15,925 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 07:13:15,936 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 07:13:15,960 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 07:13:15,962 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 07:13:15,963 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 07:13:15,964 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 07:13:15,967 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 07:13:15,968 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 07:13:15,969 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 07:13:15,969 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 07:13:15,971 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 07:13:15,972 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 07:13:15,973 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 07:13:15,974 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 07:13:15,975 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 07:13:15,975 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 07:13:15,976 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 07:13:15,976 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 07:13:15,977 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 07:13:15,977 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 07:13:15,978 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 07:13:15,978 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 07:13:15,979 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 07:13:15,979 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 07:13:15,981 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 07:13:15,982 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 07:13:15,983 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 07:13:15,983 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 07:13:15,985 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 07:13:15,987 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 07:13:15,989 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 07:13:15,990 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 07:13:15,992 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 07:13:15,993 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 07:13:16,011 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 07:13:16,032 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 07:13:16,035 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 07:13:16,036 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 07:13:16,037 main DEBUG createAppenders(={Console}) 2024-11-13 07:13:16,038 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7a6d7e92 initialized 2024-11-13 07:13:16,039 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7a6d7e92 2024-11-13 07:13:16,040 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7a6d7e92 OK. 2024-11-13 07:13:16,043 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 07:13:16,044 main DEBUG OutputStream closed 2024-11-13 07:13:16,045 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 07:13:16,045 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 07:13:16,046 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@437da279 OK 2024-11-13 07:13:16,153 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 07:13:16,156 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 07:13:16,158 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 07:13:16,160 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 07:13:16,161 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 07:13:16,161 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 07:13:16,162 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 07:13:16,162 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 07:13:16,163 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 07:13:16,163 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 07:13:16,164 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 07:13:16,164 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 07:13:16,165 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 07:13:16,165 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 07:13:16,166 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 07:13:16,166 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 07:13:16,166 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 07:13:16,168 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 07:13:16,171 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 07:13:16,171 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@7e4204e2) with optional ClassLoader: null 2024-11-13 07:13:16,172 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 07:13:16,173 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@7e4204e2] started OK. 2024-11-13T07:13:16,189 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.procedure.TestSnapshotProcedure timeout: 13 mins 2024-11-13 07:13:16,193 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 07:13:16,193 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T07:13:16,204 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.procedure.TestSnapshotProcedureRSCrashes timeout: 13 mins 2024-11-13T07:13:16,250 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.procedure.TestSnapshotProcedureRSCrashes#testRegionServerCrashWhileTakingSnapshot Thread=11, OpenFileDescriptor=285, MaxFileDescriptor=1048576, SystemLoadAverage=844, ProcessCount=11, AvailableMemoryMB=5608 2024-11-13T07:13:16,697 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36 2024-11-13T07:13:16,700 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T07:13:16,744 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/cluster_fa0907df-988e-740e-d6e3-0a6b8ca54eeb, deleteOnExit=true 2024-11-13T07:13:16,744 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T07:13:16,746 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/test.cache.data in system properties and HBase conf 2024-11-13T07:13:16,747 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T07:13:16,747 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/hadoop.log.dir in system properties and HBase conf 2024-11-13T07:13:16,753 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T07:13:16,754 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T07:13:16,754 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T07:13:16,908 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T07:13:17,064 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T07:13:17,074 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T07:13:17,075 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T07:13:17,076 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T07:13:17,078 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T07:13:17,080 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T07:13:17,081 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T07:13:17,087 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T07:13:17,091 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T07:13:17,096 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T07:13:17,096 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/nfs.dump.dir in system properties and HBase conf 2024-11-13T07:13:17,097 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/java.io.tmpdir in system properties and HBase conf 2024-11-13T07:13:17,098 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T07:13:17,098 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T07:13:17,099 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T07:13:18,760 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-13T07:13:18,854 INFO [Time-limited test {}] log.Log(170): Logging initialized @4017ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-13T07:13:18,957 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T07:13:19,051 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T07:13:19,087 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T07:13:19,088 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T07:13:19,090 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T07:13:19,109 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T07:13:19,119 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7384c3bf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/hadoop.log.dir/,AVAILABLE} 2024-11-13T07:13:19,121 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@39a64764{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T07:13:19,370 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7f9aa94c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/java.io.tmpdir/jetty-localhost-36545-hadoop-hdfs-3_4_1-tests_jar-_-any-7162206872228078337/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T07:13:19,378 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13717375{HTTP/1.1, (http/1.1)}{localhost:36545} 2024-11-13T07:13:19,378 INFO [Time-limited test {}] server.Server(415): Started @4541ms 2024-11-13T07:13:20,314 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T07:13:20,328 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T07:13:20,338 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T07:13:20,338 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T07:13:20,339 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T07:13:20,341 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@225fb320{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/hadoop.log.dir/,AVAILABLE} 2024-11-13T07:13:20,342 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@55b4afc5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T07:13:20,490 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1722ba9a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/java.io.tmpdir/jetty-localhost-43871-hadoop-hdfs-3_4_1-tests_jar-_-any-17719235812437773103/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T07:13:20,492 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@57faae2b{HTTP/1.1, (http/1.1)}{localhost:43871} 2024-11-13T07:13:20,492 INFO [Time-limited test {}] server.Server(415): Started @5656ms 2024-11-13T07:13:20,565 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T07:13:20,900 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T07:13:20,927 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T07:13:20,946 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T07:13:20,946 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T07:13:20,946 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T07:13:20,954 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@cae7855{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/hadoop.log.dir/,AVAILABLE} 2024-11-13T07:13:20,955 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7a2b6a73{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T07:13:21,165 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@9d2632f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/java.io.tmpdir/jetty-localhost-41661-hadoop-hdfs-3_4_1-tests_jar-_-any-14369812638827546722/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T07:13:21,168 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c8e80d{HTTP/1.1, (http/1.1)}{localhost:41661} 2024-11-13T07:13:21,169 INFO [Time-limited test {}] server.Server(415): Started @6332ms 2024-11-13T07:13:21,174 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T07:13:21,356 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T07:13:21,378 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T07:13:21,404 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T07:13:21,405 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T07:13:21,405 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T07:13:21,406 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@55fcaba0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/hadoop.log.dir/,AVAILABLE} 2024-11-13T07:13:21,407 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3abb77fb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T07:13:21,592 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7e7c21d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/java.io.tmpdir/jetty-localhost-41415-hadoop-hdfs-3_4_1-tests_jar-_-any-15017508986682687889/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T07:13:21,593 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@30cac06d{HTTP/1.1, (http/1.1)}{localhost:41415} 2024-11-13T07:13:21,593 INFO [Time-limited test {}] server.Server(415): Started @6757ms 2024-11-13T07:13:21,597 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T07:13:22,618 WARN [Thread-119 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/cluster_fa0907df-988e-740e-d6e3-0a6b8ca54eeb/data/data1/current/BP-759209407-172.17.0.3-1731481998077/current, will proceed with Du for space computation calculation, 2024-11-13T07:13:22,620 WARN [Thread-120 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/cluster_fa0907df-988e-740e-d6e3-0a6b8ca54eeb/data/data2/current/BP-759209407-172.17.0.3-1731481998077/current, will proceed with Du for space computation calculation, 2024-11-13T07:13:22,769 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T07:13:22,788 WARN [Thread-131 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/cluster_fa0907df-988e-740e-d6e3-0a6b8ca54eeb/data/data3/current/BP-759209407-172.17.0.3-1731481998077/current, will proceed with Du for space computation calculation, 2024-11-13T07:13:22,789 WARN [Thread-132 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/cluster_fa0907df-988e-740e-d6e3-0a6b8ca54eeb/data/data4/current/BP-759209407-172.17.0.3-1731481998077/current, will proceed with Du for space computation calculation, 2024-11-13T07:13:22,880 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x90361d3a8c5f98c3 with lease ID 0x6a1c3f48de35cc12: Processing first storage report for DS-14e25fb3-d25b-4488-aa49-d869ee0f7b89 from datanode DatanodeRegistration(127.0.0.1:40257, datanodeUuid=813ce77b-5e63-4202-89e0-15afd447938b, infoPort=40023, infoSecurePort=0, ipcPort=37123, storageInfo=lv=-57;cid=testClusterID;nsid=1820916822;c=1731481998077) 2024-11-13T07:13:22,882 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x90361d3a8c5f98c3 with lease ID 0x6a1c3f48de35cc12: from storage DS-14e25fb3-d25b-4488-aa49-d869ee0f7b89 node DatanodeRegistration(127.0.0.1:40257, datanodeUuid=813ce77b-5e63-4202-89e0-15afd447938b, infoPort=40023, infoSecurePort=0, ipcPort=37123, storageInfo=lv=-57;cid=testClusterID;nsid=1820916822;c=1731481998077), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-11-13T07:13:22,882 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x90361d3a8c5f98c3 with lease ID 0x6a1c3f48de35cc12: Processing first storage report for DS-6cb8de75-8f40-44be-bf65-2e362d964f0c from datanode DatanodeRegistration(127.0.0.1:40257, datanodeUuid=813ce77b-5e63-4202-89e0-15afd447938b, infoPort=40023, infoSecurePort=0, ipcPort=37123, storageInfo=lv=-57;cid=testClusterID;nsid=1820916822;c=1731481998077) 2024-11-13T07:13:22,882 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x90361d3a8c5f98c3 with lease ID 0x6a1c3f48de35cc12: from storage DS-6cb8de75-8f40-44be-bf65-2e362d964f0c node DatanodeRegistration(127.0.0.1:40257, datanodeUuid=813ce77b-5e63-4202-89e0-15afd447938b, infoPort=40023, infoSecurePort=0, ipcPort=37123, storageInfo=lv=-57;cid=testClusterID;nsid=1820916822;c=1731481998077), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T07:13:22,894 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T07:13:22,929 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8fec74b851a59290 with lease ID 0x6a1c3f48de35cc13: Processing first storage report for DS-77acad21-8279-4d7b-afc4-61162fa08ba0 from datanode DatanodeRegistration(127.0.0.1:38027, datanodeUuid=a8997e61-4fa8-4cfe-b858-24f19c11cc2b, infoPort=43833, infoSecurePort=0, ipcPort=34935, storageInfo=lv=-57;cid=testClusterID;nsid=1820916822;c=1731481998077) 2024-11-13T07:13:22,930 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8fec74b851a59290 with lease ID 0x6a1c3f48de35cc13: from storage DS-77acad21-8279-4d7b-afc4-61162fa08ba0 node DatanodeRegistration(127.0.0.1:38027, datanodeUuid=a8997e61-4fa8-4cfe-b858-24f19c11cc2b, infoPort=43833, infoSecurePort=0, ipcPort=34935, storageInfo=lv=-57;cid=testClusterID;nsid=1820916822;c=1731481998077), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T07:13:22,935 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8fec74b851a59290 with lease ID 0x6a1c3f48de35cc13: Processing first storage report for DS-0973488b-44ee-4016-97d9-b4faf2410555 from datanode DatanodeRegistration(127.0.0.1:38027, datanodeUuid=a8997e61-4fa8-4cfe-b858-24f19c11cc2b, infoPort=43833, infoSecurePort=0, ipcPort=34935, storageInfo=lv=-57;cid=testClusterID;nsid=1820916822;c=1731481998077) 2024-11-13T07:13:22,935 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8fec74b851a59290 with lease ID 0x6a1c3f48de35cc13: from storage DS-0973488b-44ee-4016-97d9-b4faf2410555 node DatanodeRegistration(127.0.0.1:38027, datanodeUuid=a8997e61-4fa8-4cfe-b858-24f19c11cc2b, infoPort=43833, infoSecurePort=0, ipcPort=34935, storageInfo=lv=-57;cid=testClusterID;nsid=1820916822;c=1731481998077), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T07:13:23,121 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/cluster_fa0907df-988e-740e-d6e3-0a6b8ca54eeb/data/data5/current/BP-759209407-172.17.0.3-1731481998077/current, will proceed with Du for space computation calculation, 2024-11-13T07:13:23,136 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/cluster_fa0907df-988e-740e-d6e3-0a6b8ca54eeb/data/data6/current/BP-759209407-172.17.0.3-1731481998077/current, will proceed with Du for space computation calculation, 2024-11-13T07:13:23,255 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T07:13:23,268 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbaa55eb2c1a487d8 with lease ID 0x6a1c3f48de35cc14: Processing first storage report for DS-f0b0f387-c446-4c2b-8ab1-3fda3b8c8206 from datanode DatanodeRegistration(127.0.0.1:38559, datanodeUuid=8e69b974-87ae-475f-8e51-ecf56375f722, infoPort=35195, infoSecurePort=0, ipcPort=42985, storageInfo=lv=-57;cid=testClusterID;nsid=1820916822;c=1731481998077) 2024-11-13T07:13:23,269 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbaa55eb2c1a487d8 with lease ID 0x6a1c3f48de35cc14: from storage DS-f0b0f387-c446-4c2b-8ab1-3fda3b8c8206 node DatanodeRegistration(127.0.0.1:38559, datanodeUuid=8e69b974-87ae-475f-8e51-ecf56375f722, infoPort=35195, infoSecurePort=0, ipcPort=42985, storageInfo=lv=-57;cid=testClusterID;nsid=1820916822;c=1731481998077), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T07:13:23,269 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbaa55eb2c1a487d8 with lease ID 0x6a1c3f48de35cc14: Processing first storage report for DS-2db53368-df3a-450e-b125-ed453fc201b9 from datanode DatanodeRegistration(127.0.0.1:38559, datanodeUuid=8e69b974-87ae-475f-8e51-ecf56375f722, infoPort=35195, infoSecurePort=0, ipcPort=42985, storageInfo=lv=-57;cid=testClusterID;nsid=1820916822;c=1731481998077) 2024-11-13T07:13:23,269 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbaa55eb2c1a487d8 with lease ID 0x6a1c3f48de35cc14: from storage DS-2db53368-df3a-450e-b125-ed453fc201b9 node DatanodeRegistration(127.0.0.1:38559, datanodeUuid=8e69b974-87ae-475f-8e51-ecf56375f722, infoPort=35195, infoSecurePort=0, ipcPort=42985, storageInfo=lv=-57;cid=testClusterID;nsid=1820916822;c=1731481998077), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T07:13:23,301 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36 2024-11-13T07:13:23,521 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/cluster_fa0907df-988e-740e-d6e3-0a6b8ca54eeb/zookeeper_0, clientPort=56395, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/cluster_fa0907df-988e-740e-d6e3-0a6b8ca54eeb/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/cluster_fa0907df-988e-740e-d6e3-0a6b8ca54eeb/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T07:13:23,540 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56395 2024-11-13T07:13:23,556 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:13:23,562 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:13:24,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741825_1001 (size=7) 2024-11-13T07:13:24,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741825_1001 (size=7) 2024-11-13T07:13:24,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741825_1001 (size=7) 2024-11-13T07:13:24,169 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9 with version=8 2024-11-13T07:13:24,170 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/hbase-staging 2024-11-13T07:13:24,348 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-13T07:13:24,845 INFO [Time-limited test {}] client.ConnectionUtils(128): master/2998b885bdef:0 server-side Connection retries=45 2024-11-13T07:13:24,859 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T07:13:24,860 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T07:13:24,867 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T07:13:24,867 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T07:13:24,867 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T07:13:25,094 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T07:13:25,165 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T07:13:25,182 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T07:13:25,189 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T07:13:25,228 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1038 (auto-detected) 2024-11-13T07:13:25,230 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-13T07:13:25,256 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46239 2024-11-13T07:13:25,285 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46239 connecting to ZooKeeper ensemble=127.0.0.1:56395 2024-11-13T07:13:25,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:462390x0, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T07:13:25,401 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46239-0x101330aa3af0000 connected 2024-11-13T07:13:25,639 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:13:25,643 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:13:25,677 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T07:13:25,683 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9, hbase.cluster.distributed=false 2024-11-13T07:13:25,736 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T07:13:25,752 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46239 2024-11-13T07:13:25,756 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46239 2024-11-13T07:13:25,772 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46239 2024-11-13T07:13:25,792 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46239 2024-11-13T07:13:25,793 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46239 2024-11-13T07:13:25,985 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2998b885bdef:0 server-side Connection retries=45 2024-11-13T07:13:25,988 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T07:13:25,988 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T07:13:25,990 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T07:13:25,991 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T07:13:25,991 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T07:13:25,995 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T07:13:26,002 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T07:13:26,009 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44417 2024-11-13T07:13:26,013 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44417 connecting to ZooKeeper ensemble=127.0.0.1:56395 2024-11-13T07:13:26,016 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:13:26,026 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:13:26,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:444170x0, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T07:13:26,224 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44417-0x101330aa3af0001 connected 2024-11-13T07:13:26,224 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44417-0x101330aa3af0001, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T07:13:26,229 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T07:13:26,239 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T07:13:26,242 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44417-0x101330aa3af0001, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T07:13:26,247 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44417-0x101330aa3af0001, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T07:13:26,248 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44417 2024-11-13T07:13:26,249 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44417 2024-11-13T07:13:26,249 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44417 2024-11-13T07:13:26,254 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44417 2024-11-13T07:13:26,254 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44417 2024-11-13T07:13:26,275 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2998b885bdef:0 server-side Connection retries=45 2024-11-13T07:13:26,275 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T07:13:26,275 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T07:13:26,276 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T07:13:26,276 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T07:13:26,276 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T07:13:26,277 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T07:13:26,277 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T07:13:26,278 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36075 2024-11-13T07:13:26,280 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36075 connecting to ZooKeeper ensemble=127.0.0.1:56395 2024-11-13T07:13:26,282 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:13:26,285 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:13:26,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:360750x0, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T07:13:26,893 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36075-0x101330aa3af0002 connected 2024-11-13T07:13:26,893 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36075-0x101330aa3af0002, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T07:13:26,893 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T07:13:26,895 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T07:13:26,896 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36075-0x101330aa3af0002, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T07:13:26,906 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36075-0x101330aa3af0002, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T07:13:26,909 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36075 2024-11-13T07:13:26,909 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36075 2024-11-13T07:13:26,912 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36075 2024-11-13T07:13:26,914 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36075 2024-11-13T07:13:26,915 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36075 2024-11-13T07:13:26,938 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2998b885bdef:0 server-side Connection retries=45 2024-11-13T07:13:26,938 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T07:13:26,939 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T07:13:26,939 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T07:13:26,939 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T07:13:26,939 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T07:13:26,940 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T07:13:26,940 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T07:13:26,941 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37717 2024-11-13T07:13:26,944 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37717 connecting to ZooKeeper ensemble=127.0.0.1:56395 2024-11-13T07:13:26,945 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:13:26,949 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:13:26,960 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:377170x0, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T07:13:26,961 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:377170x0, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T07:13:26,962 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37717-0x101330aa3af0003 connected 2024-11-13T07:13:26,962 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T07:13:26,963 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T07:13:26,965 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37717-0x101330aa3af0003, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T07:13:26,967 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37717-0x101330aa3af0003, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T07:13:26,976 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37717 2024-11-13T07:13:26,977 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37717 2024-11-13T07:13:26,988 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37717 2024-11-13T07:13:26,994 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37717 2024-11-13T07:13:26,995 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37717 2024-11-13T07:13:27,012 DEBUG [M:0;2998b885bdef:46239 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;2998b885bdef:46239 2024-11-13T07:13:27,013 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/2998b885bdef,46239,1731482004610 2024-11-13T07:13:27,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101330aa3af0001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T07:13:27,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36075-0x101330aa3af0002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T07:13:27,024 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37717-0x101330aa3af0003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T07:13:27,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T07:13:27,028 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/2998b885bdef,46239,1731482004610 2024-11-13T07:13:27,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101330aa3af0001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T07:13:27,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36075-0x101330aa3af0002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T07:13:27,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36075-0x101330aa3af0002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:13:27,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:13:27,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101330aa3af0001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:13:27,068 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37717-0x101330aa3af0003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T07:13:27,068 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37717-0x101330aa3af0003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:13:27,070 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T07:13:27,072 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/2998b885bdef,46239,1731482004610 from backup master directory 2024-11-13T07:13:27,087 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37717-0x101330aa3af0003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T07:13:27,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101330aa3af0001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T07:13:27,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36075-0x101330aa3af0002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T07:13:27,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/2998b885bdef,46239,1731482004610 2024-11-13T07:13:27,100 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T07:13:27,101 WARN [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T07:13:27,101 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=2998b885bdef,46239,1731482004610 2024-11-13T07:13:27,103 INFO [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-13T07:13:27,107 INFO [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-13T07:13:27,195 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/hbase.id] with ID: 05a88896-60ba-480c-907e-788ba4493ab2 2024-11-13T07:13:27,195 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/.tmp/hbase.id 2024-11-13T07:13:27,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741826_1002 (size=42) 2024-11-13T07:13:27,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741826_1002 (size=42) 2024-11-13T07:13:27,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741826_1002 (size=42) 2024-11-13T07:13:27,255 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/.tmp/hbase.id]:[hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/hbase.id] 2024-11-13T07:13:27,350 INFO [master/2998b885bdef:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:13:27,355 INFO [master/2998b885bdef:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T07:13:27,383 INFO [master/2998b885bdef:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 26ms. 2024-11-13T07:13:27,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36075-0x101330aa3af0002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:13:27,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101330aa3af0001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:13:27,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:13:27,400 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37717-0x101330aa3af0003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:13:27,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741827_1003 (size=196) 2024-11-13T07:13:27,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741827_1003 (size=196) 2024-11-13T07:13:27,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741827_1003 (size=196) 2024-11-13T07:13:27,539 INFO [master/2998b885bdef:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T07:13:27,554 INFO [master/2998b885bdef:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T07:13:27,580 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:13:27,584 INFO [master/2998b885bdef:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:13:27,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741828_1004 (size=1189) 2024-11-13T07:13:27,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741828_1004 (size=1189) 2024-11-13T07:13:27,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741828_1004 (size=1189) 2024-11-13T07:13:27,681 INFO [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/MasterData/data/master/store 2024-11-13T07:13:27,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741829_1005 (size=34) 2024-11-13T07:13:27,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741829_1005 (size=34) 2024-11-13T07:13:27,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741829_1005 (size=34) 2024-11-13T07:13:27,733 INFO [master/2998b885bdef:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T07:13:27,738 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:13:27,740 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T07:13:27,740 INFO [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T07:13:27,741 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T07:13:27,744 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T07:13:27,744 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T07:13:27,745 INFO [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T07:13:27,747 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731482007740Disabling compacts and flushes for region at 1731482007740Disabling writes for close at 1731482007744 (+4 ms)Writing region close event to WAL at 1731482007745 (+1 ms)Closed at 1731482007745 2024-11-13T07:13:27,753 WARN [master/2998b885bdef:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/MasterData/data/master/store/.initializing 2024-11-13T07:13:27,753 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/MasterData/WALs/2998b885bdef,46239,1731482004610 2024-11-13T07:13:27,777 INFO [master/2998b885bdef:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T07:13:27,816 INFO [master/2998b885bdef:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2998b885bdef%2C46239%2C1731482004610, suffix=, logDir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/MasterData/WALs/2998b885bdef,46239,1731482004610, archiveDir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/MasterData/oldWALs, maxLogs=10 2024-11-13T07:13:27,849 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/MasterData/WALs/2998b885bdef,46239,1731482004610/2998b885bdef%2C46239%2C1731482004610.1731482007822, exclude list is [], retry=0 2024-11-13T07:13:27,881 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38027,DS-77acad21-8279-4d7b-afc4-61162fa08ba0,DISK] 2024-11-13T07:13:27,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38559,DS-f0b0f387-c446-4c2b-8ab1-3fda3b8c8206,DISK] 2024-11-13T07:13:27,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40257,DS-14e25fb3-d25b-4488-aa49-d869ee0f7b89,DISK] 2024-11-13T07:13:27,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-13T07:13:27,945 INFO [master/2998b885bdef:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/MasterData/WALs/2998b885bdef,46239,1731482004610/2998b885bdef%2C46239%2C1731482004610.1731482007822 2024-11-13T07:13:27,949 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40023:40023),(127.0.0.1/127.0.0.1:35195:35195),(127.0.0.1/127.0.0.1:43833:43833)] 2024-11-13T07:13:27,949 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:13:27,950 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:13:27,955 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:13:27,956 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:13:28,045 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:13:28,096 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T07:13:28,102 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:13:28,113 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:13:28,114 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:13:28,124 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T07:13:28,124 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:13:28,126 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:13:28,127 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:13:28,149 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T07:13:28,152 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:13:28,159 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:13:28,164 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:13:28,175 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T07:13:28,175 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:13:28,178 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:13:28,179 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:13:28,191 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:13:28,197 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:13:28,212 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:13:28,213 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:13:28,218 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T07:13:28,233 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:13:28,275 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:13:28,279 INFO [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73125922, jitterRate=0.08966115117073059}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T07:13:28,295 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731482007976Initializing all the Stores at 1731482007989 (+13 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731482007991 (+2 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482007997 (+6 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482007998 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482007998Cleaning up temporary data from old regions at 1731482008213 (+215 ms)Region opened successfully at 1731482008295 (+82 ms) 2024-11-13T07:13:28,304 INFO [master/2998b885bdef:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T07:13:28,435 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b40496, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2998b885bdef/172.17.0.3:0 2024-11-13T07:13:28,521 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T07:13:28,540 INFO [master/2998b885bdef:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T07:13:28,540 INFO [master/2998b885bdef:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T07:13:28,545 INFO [master/2998b885bdef:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T07:13:28,553 INFO [master/2998b885bdef:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 8 msec 2024-11-13T07:13:28,571 INFO [master/2998b885bdef:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 17 msec 2024-11-13T07:13:28,571 INFO [master/2998b885bdef:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=128, operationDelay=10000 2024-11-13T07:13:28,668 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T07:13:28,690 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T07:13:28,758 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T07:13:28,761 INFO [master/2998b885bdef:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T07:13:28,775 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T07:13:28,793 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T07:13:28,796 INFO [master/2998b885bdef:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T07:13:28,816 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T07:13:28,848 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T07:13:28,859 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T07:13:28,889 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T07:13:28,939 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T07:13:28,967 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T07:13:28,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36075-0x101330aa3af0002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T07:13:28,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36075-0x101330aa3af0002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:13:28,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T07:13:28,984 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37717-0x101330aa3af0003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T07:13:28,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:13:28,984 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37717-0x101330aa3af0003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:13:28,984 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101330aa3af0001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T07:13:28,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101330aa3af0001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:13:28,992 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=2998b885bdef,46239,1731482004610, sessionid=0x101330aa3af0000, setting cluster-up flag (Was=false) 2024-11-13T07:13:29,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101330aa3af0001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:13:29,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36075-0x101330aa3af0002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:13:29,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:13:29,041 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37717-0x101330aa3af0003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:13:29,200 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T07:13:29,222 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2998b885bdef,46239,1731482004610 2024-11-13T07:13:29,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36075-0x101330aa3af0002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:13:29,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101330aa3af0001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:13:29,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:13:29,280 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37717-0x101330aa3af0003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:13:29,364 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T07:13:29,385 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2998b885bdef,46239,1731482004610 2024-11-13T07:13:29,407 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T07:13:29,442 INFO [RS:0;2998b885bdef:44417 {}] regionserver.HRegionServer(746): ClusterId : 05a88896-60ba-480c-907e-788ba4493ab2 2024-11-13T07:13:29,446 DEBUG [RS:0;2998b885bdef:44417 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T07:13:29,466 INFO [RS:2;2998b885bdef:37717 {}] regionserver.HRegionServer(746): ClusterId : 05a88896-60ba-480c-907e-788ba4493ab2 2024-11-13T07:13:29,466 DEBUG [RS:2;2998b885bdef:37717 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T07:13:29,473 INFO [RS:1;2998b885bdef:36075 {}] regionserver.HRegionServer(746): ClusterId : 05a88896-60ba-480c-907e-788ba4493ab2 2024-11-13T07:13:29,473 DEBUG [RS:1;2998b885bdef:36075 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T07:13:29,594 DEBUG [RS:0;2998b885bdef:44417 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T07:13:29,594 DEBUG [RS:0;2998b885bdef:44417 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T07:13:29,598 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T07:13:29,602 DEBUG [RS:2;2998b885bdef:37717 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T07:13:29,602 DEBUG [RS:2;2998b885bdef:37717 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T07:13:29,620 INFO [master/2998b885bdef:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T07:13:29,629 DEBUG [RS:1;2998b885bdef:36075 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T07:13:29,629 DEBUG [RS:1;2998b885bdef:36075 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T07:13:29,632 DEBUG [RS:2;2998b885bdef:37717 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T07:13:29,633 DEBUG [RS:2;2998b885bdef:37717 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1abb92f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2998b885bdef/172.17.0.3:0 2024-11-13T07:13:29,648 DEBUG [RS:0;2998b885bdef:44417 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T07:13:29,649 DEBUG [RS:0;2998b885bdef:44417 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@342cfbb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2998b885bdef/172.17.0.3:0 2024-11-13T07:13:29,656 INFO [master/2998b885bdef:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T07:13:29,658 DEBUG [RS:1;2998b885bdef:36075 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T07:13:29,658 DEBUG [RS:1;2998b885bdef:36075 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a04dfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2998b885bdef/172.17.0.3:0 2024-11-13T07:13:29,681 DEBUG [RS:2;2998b885bdef:37717 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;2998b885bdef:37717 2024-11-13T07:13:29,683 DEBUG [RS:0;2998b885bdef:44417 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;2998b885bdef:44417 2024-11-13T07:13:29,691 INFO [RS:0;2998b885bdef:44417 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T07:13:29,695 INFO [RS:2;2998b885bdef:37717 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T07:13:29,695 INFO [RS:2;2998b885bdef:37717 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T07:13:29,696 DEBUG [RS:2;2998b885bdef:37717 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T07:13:29,692 INFO [RS:0;2998b885bdef:44417 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T07:13:29,696 DEBUG [RS:0;2998b885bdef:44417 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T07:13:29,696 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 2998b885bdef,46239,1731482004610 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T07:13:29,694 DEBUG [RS:1;2998b885bdef:36075 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;2998b885bdef:36075 2024-11-13T07:13:29,702 INFO [RS:1;2998b885bdef:36075 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T07:13:29,703 INFO [RS:1;2998b885bdef:36075 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T07:13:29,708 DEBUG [RS:1;2998b885bdef:36075 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T07:13:29,718 INFO [RS:1;2998b885bdef:36075 {}] regionserver.HRegionServer(2659): reportForDuty to master=2998b885bdef,46239,1731482004610 with port=36075, startcode=1731482006274 2024-11-13T07:13:29,718 INFO [RS:0;2998b885bdef:44417 {}] regionserver.HRegionServer(2659): reportForDuty to master=2998b885bdef,46239,1731482004610 with port=44417, startcode=1731482005920 2024-11-13T07:13:29,724 INFO [RS:2;2998b885bdef:37717 {}] regionserver.HRegionServer(2659): reportForDuty to master=2998b885bdef,46239,1731482004610 with port=37717, startcode=1731482006937 2024-11-13T07:13:29,739 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/2998b885bdef:0, corePoolSize=5, maxPoolSize=5 2024-11-13T07:13:29,739 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/2998b885bdef:0, corePoolSize=5, maxPoolSize=5 2024-11-13T07:13:29,739 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/2998b885bdef:0, corePoolSize=5, maxPoolSize=5 2024-11-13T07:13:29,740 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/2998b885bdef:0, corePoolSize=5, maxPoolSize=5 2024-11-13T07:13:29,740 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/2998b885bdef:0, corePoolSize=10, maxPoolSize=10 2024-11-13T07:13:29,740 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:29,740 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/2998b885bdef:0, corePoolSize=2, maxPoolSize=2 2024-11-13T07:13:29,741 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:29,756 DEBUG [RS:1;2998b885bdef:36075 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T07:13:29,756 DEBUG [RS:0;2998b885bdef:44417 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T07:13:29,765 DEBUG [RS:2;2998b885bdef:37717 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T07:13:29,805 INFO [master/2998b885bdef:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731482039805 2024-11-13T07:13:29,807 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T07:13:29,809 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T07:13:29,814 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T07:13:29,814 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T07:13:29,814 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T07:13:29,815 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T07:13:29,815 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T07:13:29,815 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T07:13:29,829 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:13:29,829 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T07:13:29,828 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:29,848 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T07:13:29,850 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T07:13:29,851 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T07:13:29,879 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T07:13:29,879 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T07:13:29,892 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/2998b885bdef:0:becomeActiveMaster-HFileCleaner.large.0-1731482009881,5,FailOnTimeoutGroup] 2024-11-13T07:13:29,897 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53979, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T07:13:29,900 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51079, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T07:13:29,903 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57183, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T07:13:29,908 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/2998b885bdef:0:becomeActiveMaster-HFileCleaner.small.0-1731482009902,5,FailOnTimeoutGroup] 2024-11-13T07:13:29,908 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:29,909 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T07:13:29,910 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:29,911 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:29,911 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46239 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T07:13:29,965 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46239 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2998b885bdef,37717,1731482006937 2024-11-13T07:13:29,969 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46239 {}] master.ServerManager(517): Registering regionserver=2998b885bdef,37717,1731482006937 2024-11-13T07:13:30,000 DEBUG [RS:2;2998b885bdef:37717 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9 2024-11-13T07:13:30,000 DEBUG [RS:2;2998b885bdef:37717 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41763 2024-11-13T07:13:30,000 DEBUG [RS:2;2998b885bdef:37717 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T07:13:29,997 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46239 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2998b885bdef,36075,1731482006274 2024-11-13T07:13:30,001 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46239 {}] master.ServerManager(517): Registering regionserver=2998b885bdef,36075,1731482006274 2024-11-13T07:13:30,027 DEBUG [RS:2;2998b885bdef:37717 {}] zookeeper.ZKUtil(111): regionserver:37717-0x101330aa3af0003, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2998b885bdef,37717,1731482006937 2024-11-13T07:13:30,027 WARN [RS:2;2998b885bdef:37717 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T07:13:30,028 INFO [RS:2;2998b885bdef:37717 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:13:30,028 DEBUG [RS:2;2998b885bdef:37717 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,37717,1731482006937 2024-11-13T07:13:30,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T07:13:30,033 DEBUG [RS:1;2998b885bdef:36075 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9 2024-11-13T07:13:30,033 DEBUG [RS:1;2998b885bdef:36075 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41763 2024-11-13T07:13:30,033 DEBUG [RS:1;2998b885bdef:36075 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T07:13:30,044 DEBUG [RS:0;2998b885bdef:44417 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T07:13:30,045 WARN [RS:0;2998b885bdef:44417 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T07:13:30,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741831_1007 (size=1321) 2024-11-13T07:13:30,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741831_1007 (size=1321) 2024-11-13T07:13:30,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741831_1007 (size=1321) 2024-11-13T07:13:30,060 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T07:13:30,061 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9 2024-11-13T07:13:30,109 DEBUG [RS:1;2998b885bdef:36075 {}] zookeeper.ZKUtil(111): regionserver:36075-0x101330aa3af0002, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2998b885bdef,36075,1731482006274 2024-11-13T07:13:30,109 WARN [RS:1;2998b885bdef:36075 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T07:13:30,110 INFO [RS:1;2998b885bdef:36075 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:13:30,110 DEBUG [RS:1;2998b885bdef:36075 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274 2024-11-13T07:13:30,110 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2998b885bdef,37717,1731482006937] 2024-11-13T07:13:30,110 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2998b885bdef,36075,1731482006274] 2024-11-13T07:13:30,152 INFO [RS:0;2998b885bdef:44417 {}] regionserver.HRegionServer(2659): reportForDuty to master=2998b885bdef,46239,1731482004610 with port=44417, startcode=1731482005920 2024-11-13T07:13:30,155 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46239 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2998b885bdef,44417,1731482005920 2024-11-13T07:13:30,155 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46239 {}] master.ServerManager(517): Registering regionserver=2998b885bdef,44417,1731482005920 2024-11-13T07:13:30,167 DEBUG [RS:0;2998b885bdef:44417 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9 2024-11-13T07:13:30,167 DEBUG [RS:0;2998b885bdef:44417 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41763 2024-11-13T07:13:30,168 DEBUG [RS:0;2998b885bdef:44417 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T07:13:30,168 INFO [RS:1;2998b885bdef:36075 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T07:13:30,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T07:13:30,195 DEBUG [RS:0;2998b885bdef:44417 {}] zookeeper.ZKUtil(111): regionserver:44417-0x101330aa3af0001, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2998b885bdef,44417,1731482005920 2024-11-13T07:13:30,195 WARN [RS:0;2998b885bdef:44417 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T07:13:30,196 INFO [RS:0;2998b885bdef:44417 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:13:30,196 DEBUG [RS:0;2998b885bdef:44417 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,44417,1731482005920 2024-11-13T07:13:30,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741832_1008 (size=32) 2024-11-13T07:13:30,200 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2998b885bdef,44417,1731482005920] 2024-11-13T07:13:30,202 INFO [RS:2;2998b885bdef:37717 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T07:13:30,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741832_1008 (size=32) 2024-11-13T07:13:30,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741832_1008 (size=32) 2024-11-13T07:13:30,210 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:13:30,223 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T07:13:30,239 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T07:13:30,239 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:13:30,253 INFO [RS:0;2998b885bdef:44417 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T07:13:30,253 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:13:30,254 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T07:13:30,257 INFO [RS:1;2998b885bdef:36075 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T07:13:30,272 INFO [RS:2;2998b885bdef:37717 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T07:13:30,272 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T07:13:30,273 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:13:30,286 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:13:30,286 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T07:13:30,291 INFO [RS:2;2998b885bdef:37717 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T07:13:30,291 INFO [RS:2;2998b885bdef:37717 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:30,292 INFO [RS:0;2998b885bdef:44417 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T07:13:30,292 INFO [RS:1;2998b885bdef:36075 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T07:13:30,292 INFO [RS:1;2998b885bdef:36075 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:30,297 INFO [RS:0;2998b885bdef:44417 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T07:13:30,297 INFO [RS:0;2998b885bdef:44417 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:30,303 INFO [RS:1;2998b885bdef:36075 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T07:13:30,308 INFO [RS:2;2998b885bdef:37717 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T07:13:30,312 INFO [RS:0;2998b885bdef:44417 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T07:13:30,327 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T07:13:30,327 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:13:30,329 INFO [RS:1;2998b885bdef:36075 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T07:13:30,331 INFO [RS:1;2998b885bdef:36075 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:30,332 INFO [RS:2;2998b885bdef:37717 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T07:13:30,332 INFO [RS:2;2998b885bdef:37717 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:30,333 DEBUG [RS:2;2998b885bdef:37717 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,333 DEBUG [RS:2;2998b885bdef:37717 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,333 DEBUG [RS:2;2998b885bdef:37717 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,333 DEBUG [RS:2;2998b885bdef:37717 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,334 DEBUG [RS:2;2998b885bdef:37717 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,334 DEBUG [RS:2;2998b885bdef:37717 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0, corePoolSize=2, maxPoolSize=2 2024-11-13T07:13:30,334 DEBUG [RS:2;2998b885bdef:37717 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,334 DEBUG [RS:2;2998b885bdef:37717 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,334 DEBUG [RS:2;2998b885bdef:37717 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,334 DEBUG [RS:2;2998b885bdef:37717 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,335 DEBUG [RS:2;2998b885bdef:37717 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,335 INFO [RS:0;2998b885bdef:44417 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T07:13:30,335 DEBUG [RS:2;2998b885bdef:37717 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,335 DEBUG [RS:2;2998b885bdef:37717 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0, corePoolSize=3, maxPoolSize=3 2024-11-13T07:13:30,335 INFO [RS:0;2998b885bdef:44417 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:30,335 DEBUG [RS:2;2998b885bdef:37717 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2998b885bdef:0, corePoolSize=3, maxPoolSize=3 2024-11-13T07:13:30,335 DEBUG [RS:0;2998b885bdef:44417 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,335 DEBUG [RS:0;2998b885bdef:44417 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,335 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:13:30,336 DEBUG [RS:0;2998b885bdef:44417 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,336 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T07:13:30,336 DEBUG [RS:0;2998b885bdef:44417 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,336 DEBUG [RS:0;2998b885bdef:44417 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,336 DEBUG [RS:0;2998b885bdef:44417 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0, corePoolSize=2, maxPoolSize=2 2024-11-13T07:13:30,336 DEBUG [RS:0;2998b885bdef:44417 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,336 DEBUG [RS:0;2998b885bdef:44417 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,337 DEBUG [RS:0;2998b885bdef:44417 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,337 DEBUG [RS:0;2998b885bdef:44417 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,337 DEBUG [RS:0;2998b885bdef:44417 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,337 DEBUG [RS:0;2998b885bdef:44417 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,337 DEBUG [RS:0;2998b885bdef:44417 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0, corePoolSize=3, maxPoolSize=3 2024-11-13T07:13:30,337 DEBUG [RS:0;2998b885bdef:44417 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2998b885bdef:0, corePoolSize=3, maxPoolSize=3 2024-11-13T07:13:30,340 DEBUG [RS:1;2998b885bdef:36075 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,340 DEBUG [RS:1;2998b885bdef:36075 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,340 DEBUG [RS:1;2998b885bdef:36075 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,341 DEBUG [RS:1;2998b885bdef:36075 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,341 DEBUG [RS:1;2998b885bdef:36075 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,341 DEBUG [RS:1;2998b885bdef:36075 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0, corePoolSize=2, maxPoolSize=2 2024-11-13T07:13:30,341 DEBUG [RS:1;2998b885bdef:36075 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,341 DEBUG [RS:1;2998b885bdef:36075 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,341 DEBUG [RS:1;2998b885bdef:36075 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,342 DEBUG [RS:1;2998b885bdef:36075 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,342 DEBUG [RS:1;2998b885bdef:36075 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,342 DEBUG [RS:1;2998b885bdef:36075 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:13:30,342 DEBUG [RS:1;2998b885bdef:36075 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0, corePoolSize=3, maxPoolSize=3 2024-11-13T07:13:30,342 DEBUG [RS:1;2998b885bdef:36075 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2998b885bdef:0, corePoolSize=3, maxPoolSize=3 2024-11-13T07:13:30,346 INFO [RS:2;2998b885bdef:37717 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:30,347 INFO [RS:2;2998b885bdef:37717 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:30,347 INFO [RS:2;2998b885bdef:37717 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:30,347 INFO [RS:2;2998b885bdef:37717 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:30,347 INFO [RS:2;2998b885bdef:37717 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:30,347 INFO [RS:2;2998b885bdef:37717 {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,37717,1731482006937-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T07:13:30,364 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T07:13:30,365 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:13:30,367 INFO [RS:1;2998b885bdef:36075 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:30,367 INFO [RS:1;2998b885bdef:36075 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:30,367 INFO [RS:1;2998b885bdef:36075 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:30,367 INFO [RS:1;2998b885bdef:36075 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:30,367 INFO [RS:1;2998b885bdef:36075 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:30,368 INFO [RS:1;2998b885bdef:36075 {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,36075,1731482006274-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T07:13:30,372 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:13:30,373 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T07:13:30,381 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740 2024-11-13T07:13:30,381 INFO [RS:0;2998b885bdef:44417 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:30,382 INFO [RS:0;2998b885bdef:44417 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:30,382 INFO [RS:0;2998b885bdef:44417 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:30,382 INFO [RS:0;2998b885bdef:44417 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:30,382 INFO [RS:0;2998b885bdef:44417 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:30,382 INFO [RS:0;2998b885bdef:44417 {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,44417,1731482005920-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T07:13:30,388 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740 2024-11-13T07:13:30,408 INFO [RS:1;2998b885bdef:36075 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T07:13:30,409 INFO [RS:2;2998b885bdef:37717 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T07:13:30,411 INFO [RS:2;2998b885bdef:37717 {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,37717,1731482006937-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:30,412 INFO [RS:1;2998b885bdef:36075 {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,36075,1731482006274-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:30,413 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T07:13:30,413 INFO [RS:1;2998b885bdef:36075 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:30,413 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T07:13:30,413 INFO [RS:1;2998b885bdef:36075 {}] regionserver.Replication(171): 2998b885bdef,36075,1731482006274 started 2024-11-13T07:13:30,413 INFO [RS:2;2998b885bdef:37717 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:30,414 INFO [RS:2;2998b885bdef:37717 {}] regionserver.Replication(171): 2998b885bdef,37717,1731482006937 started 2024-11-13T07:13:30,418 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T07:13:30,440 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T07:13:30,448 INFO [RS:0;2998b885bdef:44417 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T07:13:30,456 INFO [RS:0;2998b885bdef:44417 {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,44417,1731482005920-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:30,457 INFO [RS:0;2998b885bdef:44417 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:30,457 INFO [RS:0;2998b885bdef:44417 {}] regionserver.Replication(171): 2998b885bdef,44417,1731482005920 started 2024-11-13T07:13:30,474 INFO [RS:1;2998b885bdef:36075 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:30,475 INFO [RS:2;2998b885bdef:37717 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:30,475 INFO [RS:2;2998b885bdef:37717 {}] regionserver.HRegionServer(1482): Serving as 2998b885bdef,37717,1731482006937, RpcServer on 2998b885bdef/172.17.0.3:37717, sessionid=0x101330aa3af0003 2024-11-13T07:13:30,476 DEBUG [RS:2;2998b885bdef:37717 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T07:13:30,476 DEBUG [RS:2;2998b885bdef:37717 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2998b885bdef,37717,1731482006937 2024-11-13T07:13:30,477 DEBUG [RS:2;2998b885bdef:37717 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2998b885bdef,37717,1731482006937' 2024-11-13T07:13:30,477 DEBUG [RS:2;2998b885bdef:37717 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T07:13:30,475 INFO [RS:1;2998b885bdef:36075 {}] regionserver.HRegionServer(1482): Serving as 2998b885bdef,36075,1731482006274, RpcServer on 2998b885bdef/172.17.0.3:36075, sessionid=0x101330aa3af0002 2024-11-13T07:13:30,478 DEBUG [RS:1;2998b885bdef:36075 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T07:13:30,478 DEBUG [RS:1;2998b885bdef:36075 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2998b885bdef,36075,1731482006274 2024-11-13T07:13:30,478 DEBUG [RS:1;2998b885bdef:36075 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2998b885bdef,36075,1731482006274' 2024-11-13T07:13:30,478 DEBUG [RS:1;2998b885bdef:36075 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T07:13:30,483 DEBUG [RS:2;2998b885bdef:37717 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T07:13:30,483 DEBUG [RS:1;2998b885bdef:36075 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T07:13:30,483 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:13:30,484 DEBUG [RS:2;2998b885bdef:37717 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T07:13:30,484 DEBUG [RS:2;2998b885bdef:37717 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T07:13:30,485 DEBUG [RS:2;2998b885bdef:37717 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2998b885bdef,37717,1731482006937 2024-11-13T07:13:30,485 DEBUG [RS:2;2998b885bdef:37717 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2998b885bdef,37717,1731482006937' 2024-11-13T07:13:30,485 DEBUG [RS:2;2998b885bdef:37717 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T07:13:30,488 DEBUG [RS:1;2998b885bdef:36075 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T07:13:30,488 DEBUG [RS:1;2998b885bdef:36075 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T07:13:30,489 DEBUG [RS:1;2998b885bdef:36075 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2998b885bdef,36075,1731482006274 2024-11-13T07:13:30,489 DEBUG [RS:1;2998b885bdef:36075 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2998b885bdef,36075,1731482006274' 2024-11-13T07:13:30,489 DEBUG [RS:1;2998b885bdef:36075 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T07:13:30,492 DEBUG [RS:2;2998b885bdef:37717 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T07:13:30,497 DEBUG [RS:1;2998b885bdef:36075 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T07:13:30,498 DEBUG [RS:2;2998b885bdef:37717 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T07:13:30,498 INFO [RS:2;2998b885bdef:37717 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T07:13:30,498 INFO [RS:2;2998b885bdef:37717 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T07:13:30,498 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67465087, jitterRate=0.005308136343955994}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T07:13:30,500 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731482010210Initializing all the Stores at 1731482010222 (+12 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731482010223 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731482010223Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482010226 (+3 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731482010227 (+1 ms)Cleaning up temporary data from old regions at 1731482010413 (+186 ms)Region opened successfully at 1731482010500 (+87 ms) 2024-11-13T07:13:30,500 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T07:13:30,501 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T07:13:30,501 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T07:13:30,501 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T07:13:30,501 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T07:13:30,504 DEBUG [RS:1;2998b885bdef:36075 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T07:13:30,504 INFO [RS:1;2998b885bdef:36075 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T07:13:30,504 INFO [RS:1;2998b885bdef:36075 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T07:13:30,509 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T07:13:30,509 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731482010500Disabling compacts and flushes for region at 1731482010500Disabling writes for close at 1731482010501 (+1 ms)Writing region close event to WAL at 1731482010508 (+7 ms)Closed at 1731482010509 (+1 ms) 2024-11-13T07:13:30,511 INFO [RS:0;2998b885bdef:44417 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:30,515 INFO [RS:0;2998b885bdef:44417 {}] regionserver.HRegionServer(1482): Serving as 2998b885bdef,44417,1731482005920, RpcServer on 2998b885bdef/172.17.0.3:44417, sessionid=0x101330aa3af0001 2024-11-13T07:13:30,516 DEBUG [RS:0;2998b885bdef:44417 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T07:13:30,516 DEBUG [RS:0;2998b885bdef:44417 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2998b885bdef,44417,1731482005920 2024-11-13T07:13:30,516 DEBUG [RS:0;2998b885bdef:44417 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2998b885bdef,44417,1731482005920' 2024-11-13T07:13:30,517 DEBUG [RS:0;2998b885bdef:44417 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T07:13:30,521 DEBUG [RS:0;2998b885bdef:44417 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T07:13:30,524 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T07:13:30,525 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T07:13:30,532 DEBUG [RS:0;2998b885bdef:44417 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T07:13:30,534 DEBUG [RS:0;2998b885bdef:44417 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T07:13:30,535 DEBUG [RS:0;2998b885bdef:44417 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2998b885bdef,44417,1731482005920 2024-11-13T07:13:30,536 DEBUG [RS:0;2998b885bdef:44417 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2998b885bdef,44417,1731482005920' 2024-11-13T07:13:30,536 DEBUG [RS:0;2998b885bdef:44417 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T07:13:30,542 DEBUG [RS:0;2998b885bdef:44417 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T07:13:30,543 DEBUG [RS:0;2998b885bdef:44417 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T07:13:30,543 INFO [RS:0;2998b885bdef:44417 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T07:13:30,543 INFO [RS:0;2998b885bdef:44417 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T07:13:30,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T07:13:30,585 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T07:13:30,600 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T07:13:30,607 INFO [RS:2;2998b885bdef:37717 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T07:13:30,609 INFO [RS:1;2998b885bdef:36075 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T07:13:30,622 INFO [RS:2;2998b885bdef:37717 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2998b885bdef%2C37717%2C1731482006937, suffix=, logDir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,37717,1731482006937, archiveDir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/oldWALs, maxLogs=32 2024-11-13T07:13:30,629 INFO [RS:1;2998b885bdef:36075 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2998b885bdef%2C36075%2C1731482006274, suffix=, logDir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274, archiveDir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/oldWALs, maxLogs=32 2024-11-13T07:13:30,649 INFO [RS:0;2998b885bdef:44417 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T07:13:30,655 DEBUG [RS:1;2998b885bdef:36075 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274/2998b885bdef%2C36075%2C1731482006274.1731482010632, exclude list is [], retry=0 2024-11-13T07:13:30,675 DEBUG [RS:2;2998b885bdef:37717 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,37717,1731482006937/2998b885bdef%2C37717%2C1731482006937.1731482010625, exclude list is [], retry=0 2024-11-13T07:13:30,682 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38559,DS-f0b0f387-c446-4c2b-8ab1-3fda3b8c8206,DISK] 2024-11-13T07:13:30,682 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38027,DS-77acad21-8279-4d7b-afc4-61162fa08ba0,DISK] 2024-11-13T07:13:30,682 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40257,DS-14e25fb3-d25b-4488-aa49-d869ee0f7b89,DISK] 2024-11-13T07:13:30,686 INFO [RS:0;2998b885bdef:44417 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2998b885bdef%2C44417%2C1731482005920, suffix=, logDir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,44417,1731482005920, archiveDir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/oldWALs, maxLogs=32 2024-11-13T07:13:30,691 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38559,DS-f0b0f387-c446-4c2b-8ab1-3fda3b8c8206,DISK] 2024-11-13T07:13:30,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38027,DS-77acad21-8279-4d7b-afc4-61162fa08ba0,DISK] 2024-11-13T07:13:30,700 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40257,DS-14e25fb3-d25b-4488-aa49-d869ee0f7b89,DISK] 2024-11-13T07:13:30,721 INFO [RS:1;2998b885bdef:36075 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274/2998b885bdef%2C36075%2C1731482006274.1731482010632 2024-11-13T07:13:30,722 DEBUG [RS:0;2998b885bdef:44417 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,44417,1731482005920/2998b885bdef%2C44417%2C1731482005920.1731482010689, exclude list is [], retry=0 2024-11-13T07:13:30,737 DEBUG [RS:1;2998b885bdef:36075 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35195:35195),(127.0.0.1/127.0.0.1:43833:43833),(127.0.0.1/127.0.0.1:40023:40023)] 2024-11-13T07:13:30,760 WARN [2998b885bdef:46239 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T07:13:30,770 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38027,DS-77acad21-8279-4d7b-afc4-61162fa08ba0,DISK] 2024-11-13T07:13:30,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40257,DS-14e25fb3-d25b-4488-aa49-d869ee0f7b89,DISK] 2024-11-13T07:13:30,772 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38559,DS-f0b0f387-c446-4c2b-8ab1-3fda3b8c8206,DISK] 2024-11-13T07:13:30,781 INFO [RS:2;2998b885bdef:37717 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,37717,1731482006937/2998b885bdef%2C37717%2C1731482006937.1731482010625 2024-11-13T07:13:30,794 DEBUG [RS:2;2998b885bdef:37717 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35195:35195),(127.0.0.1/127.0.0.1:43833:43833),(127.0.0.1/127.0.0.1:40023:40023)] 2024-11-13T07:13:30,840 INFO [RS:0;2998b885bdef:44417 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,44417,1731482005920/2998b885bdef%2C44417%2C1731482005920.1731482010689 2024-11-13T07:13:30,844 DEBUG [RS:0;2998b885bdef:44417 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43833:43833),(127.0.0.1/127.0.0.1:40023:40023),(127.0.0.1/127.0.0.1:35195:35195)] 2024-11-13T07:13:31,022 DEBUG [2998b885bdef:46239 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-13T07:13:31,049 DEBUG [2998b885bdef:46239 {}] balancer.BalancerClusterState(204): Hosts are {2998b885bdef=0} racks are {/default-rack=0} 2024-11-13T07:13:31,092 DEBUG [2998b885bdef:46239 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T07:13:31,092 DEBUG [2998b885bdef:46239 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T07:13:31,092 DEBUG [2998b885bdef:46239 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-13T07:13:31,092 DEBUG [2998b885bdef:46239 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T07:13:31,093 DEBUG [2998b885bdef:46239 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T07:13:31,093 DEBUG [2998b885bdef:46239 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T07:13:31,093 INFO [2998b885bdef:46239 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T07:13:31,093 INFO [2998b885bdef:46239 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T07:13:31,093 INFO [2998b885bdef:46239 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T07:13:31,093 DEBUG [2998b885bdef:46239 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T07:13:31,114 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2998b885bdef,36075,1731482006274 2024-11-13T07:13:31,136 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2998b885bdef,36075,1731482006274, state=OPENING 2024-11-13T07:13:31,164 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T07:13:31,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101330aa3af0001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:13:31,179 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37717-0x101330aa3af0003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:13:31,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36075-0x101330aa3af0002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:13:31,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:13:31,181 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:13:31,183 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:13:31,189 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:13:31,191 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:13:31,196 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T07:13:31,199 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2998b885bdef,36075,1731482006274}] 2024-11-13T07:13:35,161 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T07:13:35,161 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-13T07:13:41,256 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T07:13:41,259 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48635, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T07:13:41,275 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T07:13:41,276 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:13:41,276 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T07:13:41,280 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2998b885bdef%2C36075%2C1731482006274.meta, suffix=.meta, logDir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274, archiveDir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/oldWALs, maxLogs=32 2024-11-13T07:13:41,297 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274/2998b885bdef%2C36075%2C1731482006274.meta.1731482021283.meta, exclude list is [], retry=0 2024-11-13T07:13:41,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40257,DS-14e25fb3-d25b-4488-aa49-d869ee0f7b89,DISK] 2024-11-13T07:13:41,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38559,DS-f0b0f387-c446-4c2b-8ab1-3fda3b8c8206,DISK] 2024-11-13T07:13:41,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38027,DS-77acad21-8279-4d7b-afc4-61162fa08ba0,DISK] 2024-11-13T07:13:41,306 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274/2998b885bdef%2C36075%2C1731482006274.meta.1731482021283.meta 2024-11-13T07:13:41,307 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40023:40023),(127.0.0.1/127.0.0.1:35195:35195),(127.0.0.1/127.0.0.1:43833:43833)] 2024-11-13T07:13:41,307 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:13:41,309 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T07:13:41,311 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T07:13:41,316 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T07:13:41,321 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T07:13:41,321 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:13:41,321 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T07:13:41,321 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T07:13:41,324 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T07:13:41,326 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T07:13:41,326 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:13:41,327 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:13:41,327 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T07:13:41,328 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T07:13:41,328 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:13:41,329 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:13:41,329 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T07:13:41,331 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T07:13:41,331 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:13:41,332 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:13:41,332 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T07:13:41,333 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T07:13:41,333 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:13:41,334 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:13:41,334 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T07:13:41,335 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740 2024-11-13T07:13:41,338 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740 2024-11-13T07:13:41,340 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T07:13:41,340 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T07:13:41,341 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T07:13:41,344 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T07:13:41,346 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73320282, jitterRate=0.09255734086036682}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T07:13:41,347 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T07:13:41,349 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731482021322Writing region info on filesystem at 1731482021322Initializing all the Stores at 1731482021324 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731482021324Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731482021324Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482021324Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731482021324Cleaning up temporary data from old regions at 1731482021340 (+16 ms)Running coprocessor post-open hooks at 1731482021347 (+7 ms)Region opened successfully at 1731482021348 (+1 ms) 2024-11-13T07:13:41,356 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731482021246 2024-11-13T07:13:41,366 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T07:13:41,366 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T07:13:41,368 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=2998b885bdef,36075,1731482006274 2024-11-13T07:13:41,371 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2998b885bdef,36075,1731482006274, state=OPEN 2024-11-13T07:13:41,490 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37717-0x101330aa3af0003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T07:13:41,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101330aa3af0001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T07:13:41,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36075-0x101330aa3af0002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T07:13:41,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T07:13:41,491 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:13:41,491 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:13:41,491 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:13:41,491 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:13:41,491 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2998b885bdef,36075,1731482006274 2024-11-13T07:13:41,496 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T07:13:41,496 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2998b885bdef,36075,1731482006274 in 10.2930 sec 2024-11-13T07:13:41,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T07:13:41,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 10.9540 sec 2024-11-13T07:13:41,504 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T07:13:41,504 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T07:13:41,524 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T07:13:41,525 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2998b885bdef,36075,1731482006274, seqNum=-1] 2024-11-13T07:13:41,567 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T07:13:41,569 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38477, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T07:13:41,601 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 12.1160 sec 2024-11-13T07:13:41,601 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731482021601, completionTime=-1 2024-11-13T07:13:41,604 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-13T07:13:41,605 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T07:13:41,638 INFO [master/2998b885bdef:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-13T07:13:41,638 INFO [master/2998b885bdef:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731482081638 2024-11-13T07:13:41,638 INFO [master/2998b885bdef:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731482141638 2024-11-13T07:13:41,639 INFO [master/2998b885bdef:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 34 msec 2024-11-13T07:13:41,640 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-13T07:13:41,646 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,46239,1731482004610-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:41,646 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,46239,1731482004610-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:41,646 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,46239,1731482004610-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:41,647 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-2998b885bdef:46239, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:41,648 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:41,648 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:41,653 DEBUG [master/2998b885bdef:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T07:13:41,672 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 14.571sec 2024-11-13T07:13:41,673 INFO [master/2998b885bdef:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T07:13:41,674 INFO [master/2998b885bdef:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T07:13:41,675 INFO [master/2998b885bdef:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T07:13:41,675 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T07:13:41,675 INFO [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T07:13:41,676 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,46239,1731482004610-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T07:13:41,676 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,46239,1731482004610-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T07:13:41,680 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T07:13:41,681 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T07:13:41,682 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,46239,1731482004610-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T07:13:41,726 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b52e66c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T07:13:41,731 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-13T07:13:41,731 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-13T07:13:41,734 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2998b885bdef,46239,-1 for getting cluster id 2024-11-13T07:13:41,737 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T07:13:41,747 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '05a88896-60ba-480c-907e-788ba4493ab2' 2024-11-13T07:13:41,750 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T07:13:41,750 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "05a88896-60ba-480c-907e-788ba4493ab2" 2024-11-13T07:13:41,750 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5093486e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T07:13:41,750 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2998b885bdef,46239,-1] 2024-11-13T07:13:41,753 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T07:13:41,755 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:13:41,756 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T07:13:41,758 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b7152e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T07:13:41,759 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T07:13:41,765 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2998b885bdef,36075,1731482006274, seqNum=-1] 2024-11-13T07:13:41,766 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T07:13:41,767 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T07:13:41,787 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=2998b885bdef,46239,1731482004610 2024-11-13T07:13:41,791 DEBUG [Time-limited test {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731482021790 (current time:1731482021790). 2024-11-13T07:13:41,791 DEBUG [Time-limited test {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-13T07:13:41,791 DEBUG [Time-limited test {}] snapshot.SnapshotDescriptionUtils(339): Snapshot SnapshotProcedureTest VERSION not specified, setting to 2 2024-11-13T07:13:41,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@de9cb9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T07:13:41,793 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2998b885bdef,46239,-1 for getting cluster id 2024-11-13T07:13:41,794 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T07:13:41,795 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '05a88896-60ba-480c-907e-788ba4493ab2' 2024-11-13T07:13:41,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T07:13:41,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "05a88896-60ba-480c-907e-788ba4493ab2" 2024-11-13T07:13:41,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ca9cd32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T07:13:41,796 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2998b885bdef,46239,-1] 2024-11-13T07:13:41,796 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T07:13:41,796 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:13:41,797 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T07:13:41,798 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f81376e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T07:13:41,800 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T07:13:41,802 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2998b885bdef,36075,1731482006274, seqNum=-1] 2024-11-13T07:13:41,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T07:13:41,804 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T07:13:41,805 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T07:13:41,805 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.procedure.TestSnapshotProcedure.setup(TestSnapshotProcedure.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunBefores.invokeMethod(RunBefores.java:33) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T07:13:41,810 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:13:41,810 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:13:41,810 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T07:13:41,816 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T07:13:41,820 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 2998b885bdef,46239,1731482004610 2024-11-13T07:13:41,823 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@78a2313c 2024-11-13T07:13:41,824 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T07:13:41,826 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T07:13:41,835 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'SPTestTable', {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T07:13:41,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=SPTestTable 2024-11-13T07:13:41,846 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=SPTestTable execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T07:13:41,848 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "SPTestTable" procId is: 4 2024-11-13T07:13:41,849 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:13:41,851 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=SPTestTable execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T07:13:41,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T07:13:41,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741837_1013 (size=361) 2024-11-13T07:13:41,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741837_1013 (size=361) 2024-11-13T07:13:41,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741837_1013 (size=361) 2024-11-13T07:13:41,869 INFO [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c5c88a6d20592c1b4adcc2269ffce351, NAME => 'SPTestTable,,1731482021829.c5c88a6d20592c1b4adcc2269ffce351.', STARTKEY => '', ENDKEY => '19999999'}, tableDescriptor='SPTestTable', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9 2024-11-13T07:13:41,869 INFO [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 4b6c6542ce1ea1c1d3d96b4831e7c74a, NAME => 'SPTestTable,33333332,1731482021829.4b6c6542ce1ea1c1d3d96b4831e7c74a.', STARTKEY => '33333332', ENDKEY => '4ccccccb'}, tableDescriptor='SPTestTable', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9 2024-11-13T07:13:41,870 INFO [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 5f4b94b50abb029a06bde24412d357dd, NAME => 'SPTestTable,19999999,1731482021829.5f4b94b50abb029a06bde24412d357dd.', STARTKEY => '19999999', ENDKEY => '33333332'}, tableDescriptor='SPTestTable', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9 2024-11-13T07:13:41,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741838_1014 (size=62) 2024-11-13T07:13:41,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741839_1015 (size=62) 2024-11-13T07:13:41,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741838_1014 (size=62) 2024-11-13T07:13:41,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741839_1015 (size=62) 2024-11-13T07:13:41,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741839_1015 (size=62) 2024-11-13T07:13:41,887 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(898): Instantiated SPTestTable,33333332,1731482021829.4b6c6542ce1ea1c1d3d96b4831e7c74a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:13:41,887 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1722): Closing 4b6c6542ce1ea1c1d3d96b4831e7c74a, disabling compactions & flushes 2024-11-13T07:13:41,888 INFO [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1755): Closing region SPTestTable,33333332,1731482021829.4b6c6542ce1ea1c1d3d96b4831e7c74a. 2024-11-13T07:13:41,888 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,33333332,1731482021829.4b6c6542ce1ea1c1d3d96b4831e7c74a. 2024-11-13T07:13:41,888 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,33333332,1731482021829.4b6c6542ce1ea1c1d3d96b4831e7c74a. after waiting 0 ms 2024-11-13T07:13:41,888 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,33333332,1731482021829.4b6c6542ce1ea1c1d3d96b4831e7c74a. 2024-11-13T07:13:41,888 INFO [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1973): Closed SPTestTable,33333332,1731482021829.4b6c6542ce1ea1c1d3d96b4831e7c74a. 2024-11-13T07:13:41,888 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1676): Region close journal for 4b6c6542ce1ea1c1d3d96b4831e7c74a: Waiting for close lock at 1731482021887Disabling compacts and flushes for region at 1731482021887Disabling writes for close at 1731482021888 (+1 ms)Writing region close event to WAL at 1731482021888Closed at 1731482021888 2024-11-13T07:13:41,889 INFO [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 88b9c24c43eef7986e237d795658427e, NAME => 'SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e.', STARTKEY => '4ccccccb', ENDKEY => '66666664'}, tableDescriptor='SPTestTable', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9 2024-11-13T07:13:41,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741838_1014 (size=62) 2024-11-13T07:13:41,893 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(898): Instantiated SPTestTable,19999999,1731482021829.5f4b94b50abb029a06bde24412d357dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:13:41,893 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1722): Closing 5f4b94b50abb029a06bde24412d357dd, disabling compactions & flushes 2024-11-13T07:13:41,893 INFO [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1755): Closing region SPTestTable,19999999,1731482021829.5f4b94b50abb029a06bde24412d357dd. 2024-11-13T07:13:41,893 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,19999999,1731482021829.5f4b94b50abb029a06bde24412d357dd. 2024-11-13T07:13:41,893 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,19999999,1731482021829.5f4b94b50abb029a06bde24412d357dd. after waiting 0 ms 2024-11-13T07:13:41,893 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,19999999,1731482021829.5f4b94b50abb029a06bde24412d357dd. 2024-11-13T07:13:41,894 INFO [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1973): Closed SPTestTable,19999999,1731482021829.5f4b94b50abb029a06bde24412d357dd. 2024-11-13T07:13:41,894 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1676): Region close journal for 5f4b94b50abb029a06bde24412d357dd: Waiting for close lock at 1731482021893Disabling compacts and flushes for region at 1731482021893Disabling writes for close at 1731482021893Writing region close event to WAL at 1731482021894 (+1 ms)Closed at 1731482021894 2024-11-13T07:13:41,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741840_1016 (size=54) 2024-11-13T07:13:41,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741840_1016 (size=54) 2024-11-13T07:13:41,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741840_1016 (size=54) 2024-11-13T07:13:41,895 INFO [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => c880039bd4ef8e05f593aab556d82223, NAME => 'SPTestTable,66666664,1731482021829.c880039bd4ef8e05f593aab556d82223.', STARTKEY => '66666664', ENDKEY => '7ffffffd'}, tableDescriptor='SPTestTable', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9 2024-11-13T07:13:41,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741841_1017 (size=62) 2024-11-13T07:13:41,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741841_1017 (size=62) 2024-11-13T07:13:41,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741841_1017 (size=62) 2024-11-13T07:13:41,922 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(898): Instantiated SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:13:41,923 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1722): Closing 88b9c24c43eef7986e237d795658427e, disabling compactions & flushes 2024-11-13T07:13:41,923 INFO [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1755): Closing region SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e. 2024-11-13T07:13:41,923 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e. 2024-11-13T07:13:41,923 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e. after waiting 0 ms 2024-11-13T07:13:41,923 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e. 2024-11-13T07:13:41,923 INFO [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1973): Closed SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e. 2024-11-13T07:13:41,923 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1676): Region close journal for 88b9c24c43eef7986e237d795658427e: Waiting for close lock at 1731482021923Disabling compacts and flushes for region at 1731482021923Disabling writes for close at 1731482021923Writing region close event to WAL at 1731482021923Closed at 1731482021923 2024-11-13T07:13:41,924 INFO [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 6af43b5a45829fff6c10e65235eba1d3, NAME => 'SPTestTable,7ffffffd,1731482021829.6af43b5a45829fff6c10e65235eba1d3.', STARTKEY => '7ffffffd', ENDKEY => '99999996'}, tableDescriptor='SPTestTable', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9 2024-11-13T07:13:41,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741842_1018 (size=62) 2024-11-13T07:13:41,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741842_1018 (size=62) 2024-11-13T07:13:41,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741842_1018 (size=62) 2024-11-13T07:13:41,930 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(898): Instantiated SPTestTable,66666664,1731482021829.c880039bd4ef8e05f593aab556d82223.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:13:41,930 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1722): Closing c880039bd4ef8e05f593aab556d82223, disabling compactions & flushes 2024-11-13T07:13:41,930 INFO [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1755): Closing region SPTestTable,66666664,1731482021829.c880039bd4ef8e05f593aab556d82223. 2024-11-13T07:13:41,930 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,66666664,1731482021829.c880039bd4ef8e05f593aab556d82223. 2024-11-13T07:13:41,930 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,66666664,1731482021829.c880039bd4ef8e05f593aab556d82223. after waiting 0 ms 2024-11-13T07:13:41,930 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,66666664,1731482021829.c880039bd4ef8e05f593aab556d82223. 2024-11-13T07:13:41,930 INFO [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1973): Closed SPTestTable,66666664,1731482021829.c880039bd4ef8e05f593aab556d82223. 2024-11-13T07:13:41,930 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1676): Region close journal for c880039bd4ef8e05f593aab556d82223: Waiting for close lock at 1731482021930Disabling compacts and flushes for region at 1731482021930Disabling writes for close at 1731482021930Writing region close event to WAL at 1731482021930Closed at 1731482021930 2024-11-13T07:13:41,931 INFO [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => f70ed29b296742bfa400816128951d24, NAME => 'SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24.', STARTKEY => '99999996', ENDKEY => 'b333332f'}, tableDescriptor='SPTestTable', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9 2024-11-13T07:13:41,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741843_1019 (size=62) 2024-11-13T07:13:41,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741843_1019 (size=62) 2024-11-13T07:13:41,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741843_1019 (size=62) 2024-11-13T07:13:41,952 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(898): Instantiated SPTestTable,7ffffffd,1731482021829.6af43b5a45829fff6c10e65235eba1d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:13:41,952 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1722): Closing 6af43b5a45829fff6c10e65235eba1d3, disabling compactions & flushes 2024-11-13T07:13:41,952 INFO [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1755): Closing region SPTestTable,7ffffffd,1731482021829.6af43b5a45829fff6c10e65235eba1d3. 2024-11-13T07:13:41,952 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,7ffffffd,1731482021829.6af43b5a45829fff6c10e65235eba1d3. 2024-11-13T07:13:41,952 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,7ffffffd,1731482021829.6af43b5a45829fff6c10e65235eba1d3. after waiting 0 ms 2024-11-13T07:13:41,952 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,7ffffffd,1731482021829.6af43b5a45829fff6c10e65235eba1d3. 2024-11-13T07:13:41,952 INFO [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1973): Closed SPTestTable,7ffffffd,1731482021829.6af43b5a45829fff6c10e65235eba1d3. 2024-11-13T07:13:41,952 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1676): Region close journal for 6af43b5a45829fff6c10e65235eba1d3: Waiting for close lock at 1731482021952Disabling compacts and flushes for region at 1731482021952Disabling writes for close at 1731482021952Writing region close event to WAL at 1731482021952Closed at 1731482021952 2024-11-13T07:13:41,953 INFO [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => afca0ae186274ac99f7499d163ba2c4e, NAME => 'SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e.', STARTKEY => 'b333332f', ENDKEY => 'ccccccc8'}, tableDescriptor='SPTestTable', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9 2024-11-13T07:13:41,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741844_1020 (size=62) 2024-11-13T07:13:41,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741844_1020 (size=62) 2024-11-13T07:13:41,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741844_1020 (size=62) 2024-11-13T07:13:41,958 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(898): Instantiated SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:13:41,958 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1722): Closing f70ed29b296742bfa400816128951d24, disabling compactions & flushes 2024-11-13T07:13:41,958 INFO [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1755): Closing region SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24. 2024-11-13T07:13:41,958 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24. 2024-11-13T07:13:41,958 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24. after waiting 0 ms 2024-11-13T07:13:41,959 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24. 2024-11-13T07:13:41,959 INFO [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1973): Closed SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24. 2024-11-13T07:13:41,959 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1676): Region close journal for f70ed29b296742bfa400816128951d24: Waiting for close lock at 1731482021958Disabling compacts and flushes for region at 1731482021958Disabling writes for close at 1731482021959 (+1 ms)Writing region close event to WAL at 1731482021959Closed at 1731482021959 2024-11-13T07:13:41,960 INFO [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 8ee3029b3e33cc034b84b2d370d7f61d, NAME => 'SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d.', STARTKEY => 'ccccccc8', ENDKEY => 'e6666661'}, tableDescriptor='SPTestTable', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9 2024-11-13T07:13:41,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T07:13:41,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741845_1021 (size=62) 2024-11-13T07:13:41,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741845_1021 (size=62) 2024-11-13T07:13:41,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741845_1021 (size=62) 2024-11-13T07:13:41,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741846_1022 (size=62) 2024-11-13T07:13:41,987 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(898): Instantiated SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:13:41,987 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1722): Closing afca0ae186274ac99f7499d163ba2c4e, disabling compactions & flushes 2024-11-13T07:13:41,987 INFO [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1755): Closing region SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e. 2024-11-13T07:13:41,987 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e. 2024-11-13T07:13:41,987 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e. after waiting 0 ms 2024-11-13T07:13:41,987 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e. 2024-11-13T07:13:41,987 INFO [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1973): Closed SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e. 2024-11-13T07:13:41,987 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1676): Region close journal for afca0ae186274ac99f7499d163ba2c4e: Waiting for close lock at 1731482021987Disabling compacts and flushes for region at 1731482021987Disabling writes for close at 1731482021987Writing region close event to WAL at 1731482021987Closed at 1731482021987 2024-11-13T07:13:41,988 INFO [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => feb86c9c548b3a71cf3eef83fbb2adeb, NAME => 'SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb.', STARTKEY => 'e6666661', ENDKEY => ''}, tableDescriptor='SPTestTable', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9 2024-11-13T07:13:41,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741846_1022 (size=62) 2024-11-13T07:13:41,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741846_1022 (size=62) 2024-11-13T07:13:41,999 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(898): Instantiated SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:13:41,999 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1722): Closing 8ee3029b3e33cc034b84b2d370d7f61d, disabling compactions & flushes 2024-11-13T07:13:41,999 INFO [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1755): Closing region SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d. 2024-11-13T07:13:41,999 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d. 2024-11-13T07:13:41,999 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d. after waiting 0 ms 2024-11-13T07:13:41,999 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d. 2024-11-13T07:13:41,999 INFO [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1973): Closed SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d. 2024-11-13T07:13:42,000 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1676): Region close journal for 8ee3029b3e33cc034b84b2d370d7f61d: Waiting for close lock at 1731482021999Disabling compacts and flushes for region at 1731482021999Disabling writes for close at 1731482021999Writing region close event to WAL at 1731482021999Closed at 1731482021999 2024-11-13T07:13:42,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741847_1023 (size=54) 2024-11-13T07:13:42,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741847_1023 (size=54) 2024-11-13T07:13:42,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741847_1023 (size=54) 2024-11-13T07:13:42,014 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(898): Instantiated SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:13:42,014 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1722): Closing feb86c9c548b3a71cf3eef83fbb2adeb, disabling compactions & flushes 2024-11-13T07:13:42,014 INFO [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1755): Closing region SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb. 2024-11-13T07:13:42,014 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb. 2024-11-13T07:13:42,014 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb. after waiting 0 ms 2024-11-13T07:13:42,014 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb. 2024-11-13T07:13:42,015 INFO [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1973): Closed SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb. 2024-11-13T07:13:42,015 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1676): Region close journal for feb86c9c548b3a71cf3eef83fbb2adeb: Waiting for close lock at 1731482022014Disabling compacts and flushes for region at 1731482022014Disabling writes for close at 1731482022014Writing region close event to WAL at 1731482022014Closed at 1731482022014 2024-11-13T07:13:42,168 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T07:13:42,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T07:13:42,295 DEBUG [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(898): Instantiated SPTestTable,,1731482021829.c5c88a6d20592c1b4adcc2269ffce351.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:13:42,296 DEBUG [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(1722): Closing c5c88a6d20592c1b4adcc2269ffce351, disabling compactions & flushes 2024-11-13T07:13:42,296 INFO [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(1755): Closing region SPTestTable,,1731482021829.c5c88a6d20592c1b4adcc2269ffce351. 2024-11-13T07:13:42,296 DEBUG [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,,1731482021829.c5c88a6d20592c1b4adcc2269ffce351. 2024-11-13T07:13:42,296 DEBUG [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,,1731482021829.c5c88a6d20592c1b4adcc2269ffce351. after waiting 0 ms 2024-11-13T07:13:42,296 DEBUG [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,,1731482021829.c5c88a6d20592c1b4adcc2269ffce351. 2024-11-13T07:13:42,296 INFO [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(1973): Closed SPTestTable,,1731482021829.c5c88a6d20592c1b4adcc2269ffce351. 2024-11-13T07:13:42,296 DEBUG [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(1676): Region close journal for c5c88a6d20592c1b4adcc2269ffce351: Waiting for close lock at 1731482022295Disabling compacts and flushes for region at 1731482022295Disabling writes for close at 1731482022296 (+1 ms)Writing region close event to WAL at 1731482022296Closed at 1731482022296 2024-11-13T07:13:42,298 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=SPTestTable execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T07:13:42,303 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"SPTestTable,33333332,1731482021829.4b6c6542ce1ea1c1d3d96b4831e7c74a.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1731482022298"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731482022298"}]},"ts":"1731482022298"} 2024-11-13T07:13:42,304 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"SPTestTable,19999999,1731482021829.5f4b94b50abb029a06bde24412d357dd.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1731482022298"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731482022298"}]},"ts":"1731482022298"} 2024-11-13T07:13:42,304 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1731482022298"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731482022298"}]},"ts":"1731482022298"} 2024-11-13T07:13:42,304 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"SPTestTable,66666664,1731482021829.c880039bd4ef8e05f593aab556d82223.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1731482022298"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731482022298"}]},"ts":"1731482022298"} 2024-11-13T07:13:42,304 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"SPTestTable,7ffffffd,1731482021829.6af43b5a45829fff6c10e65235eba1d3.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1731482022298"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731482022298"}]},"ts":"1731482022298"} 2024-11-13T07:13:42,304 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1731482022298"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731482022298"}]},"ts":"1731482022298"} 2024-11-13T07:13:42,305 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1731482022298"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731482022298"}]},"ts":"1731482022298"} 2024-11-13T07:13:42,305 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1731482022298"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731482022298"}]},"ts":"1731482022298"} 2024-11-13T07:13:42,305 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb.","families":{"info":[{"qualifier":"regioninfo","vlen":53,"tag":[],"timestamp":"1731482022298"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731482022298"}]},"ts":"1731482022298"} 2024-11-13T07:13:42,305 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"SPTestTable,,1731482021829.c5c88a6d20592c1b4adcc2269ffce351.","families":{"info":[{"qualifier":"regioninfo","vlen":53,"tag":[],"timestamp":"1731482022298"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731482022298"}]},"ts":"1731482022298"} 2024-11-13T07:13:42,338 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 10 regions to meta. 2024-11-13T07:13:42,340 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=SPTestTable execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T07:13:42,342 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"SPTestTable","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731482022340"}]},"ts":"1731482022340"} 2024-11-13T07:13:42,346 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=SPTestTable, state=ENABLING in hbase:meta 2024-11-13T07:13:42,347 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(204): Hosts are {2998b885bdef=0} racks are {/default-rack=0} 2024-11-13T07:13:42,353 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T07:13:42,353 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T07:13:42,353 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-13T07:13:42,353 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T07:13:42,353 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T07:13:42,354 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T07:13:42,354 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T07:13:42,354 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T07:13:42,354 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T07:13:42,354 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T07:13:42,355 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=c5c88a6d20592c1b4adcc2269ffce351, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=5f4b94b50abb029a06bde24412d357dd, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=4b6c6542ce1ea1c1d3d96b4831e7c74a, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=88b9c24c43eef7986e237d795658427e, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=c880039bd4ef8e05f593aab556d82223, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=6af43b5a45829fff6c10e65235eba1d3, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=f70ed29b296742bfa400816128951d24, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=afca0ae186274ac99f7499d163ba2c4e, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=8ee3029b3e33cc034b84b2d370d7f61d, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=feb86c9c548b3a71cf3eef83fbb2adeb, ASSIGN}] 2024-11-13T07:13:42,359 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=feb86c9c548b3a71cf3eef83fbb2adeb, ASSIGN 2024-11-13T07:13:42,360 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=8ee3029b3e33cc034b84b2d370d7f61d, ASSIGN 2024-11-13T07:13:42,360 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=afca0ae186274ac99f7499d163ba2c4e, ASSIGN 2024-11-13T07:13:42,360 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=f70ed29b296742bfa400816128951d24, ASSIGN 2024-11-13T07:13:42,361 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=6af43b5a45829fff6c10e65235eba1d3, ASSIGN 2024-11-13T07:13:42,361 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=afca0ae186274ac99f7499d163ba2c4e, ASSIGN; state=OFFLINE, location=2998b885bdef,44417,1731482005920; forceNewPlan=false, retain=false 2024-11-13T07:13:42,361 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=8ee3029b3e33cc034b84b2d370d7f61d, ASSIGN; state=OFFLINE, location=2998b885bdef,37717,1731482006937; forceNewPlan=false, retain=false 2024-11-13T07:13:42,361 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=feb86c9c548b3a71cf3eef83fbb2adeb, ASSIGN; state=OFFLINE, location=2998b885bdef,36075,1731482006274; forceNewPlan=false, retain=false 2024-11-13T07:13:42,361 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=f70ed29b296742bfa400816128951d24, ASSIGN; state=OFFLINE, location=2998b885bdef,36075,1731482006274; forceNewPlan=false, retain=false 2024-11-13T07:13:42,362 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=6af43b5a45829fff6c10e65235eba1d3, ASSIGN; state=OFFLINE, location=2998b885bdef,37717,1731482006937; forceNewPlan=false, retain=false 2024-11-13T07:13:42,366 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=4b6c6542ce1ea1c1d3d96b4831e7c74a, ASSIGN 2024-11-13T07:13:42,366 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=88b9c24c43eef7986e237d795658427e, ASSIGN 2024-11-13T07:13:42,367 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=4b6c6542ce1ea1c1d3d96b4831e7c74a, ASSIGN; state=OFFLINE, location=2998b885bdef,44417,1731482005920; forceNewPlan=false, retain=false 2024-11-13T07:13:42,367 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=c880039bd4ef8e05f593aab556d82223, ASSIGN 2024-11-13T07:13:42,367 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=88b9c24c43eef7986e237d795658427e, ASSIGN; state=OFFLINE, location=2998b885bdef,36075,1731482006274; forceNewPlan=false, retain=false 2024-11-13T07:13:42,368 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=5f4b94b50abb029a06bde24412d357dd, ASSIGN 2024-11-13T07:13:42,368 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=c5c88a6d20592c1b4adcc2269ffce351, ASSIGN 2024-11-13T07:13:42,369 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=c880039bd4ef8e05f593aab556d82223, ASSIGN; state=OFFLINE, location=2998b885bdef,44417,1731482005920; forceNewPlan=false, retain=false 2024-11-13T07:13:42,369 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=5f4b94b50abb029a06bde24412d357dd, ASSIGN; state=OFFLINE, location=2998b885bdef,37717,1731482006937; forceNewPlan=false, retain=false 2024-11-13T07:13:42,369 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=c5c88a6d20592c1b4adcc2269ffce351, ASSIGN; state=OFFLINE, location=2998b885bdef,44417,1731482005920; forceNewPlan=false, retain=false 2024-11-13T07:13:42,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T07:13:42,514 INFO [2998b885bdef:46239 {}] balancer.BaseLoadBalancer(388): Reassigned 10 regions. 10 retained the pre-restart assignment. 2024-11-13T07:13:42,515 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=c880039bd4ef8e05f593aab556d82223, regionState=OPENING, regionLocation=2998b885bdef,44417,1731482005920 2024-11-13T07:13:42,515 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=afca0ae186274ac99f7499d163ba2c4e, regionState=OPENING, regionLocation=2998b885bdef,44417,1731482005920 2024-11-13T07:13:42,515 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=4b6c6542ce1ea1c1d3d96b4831e7c74a, regionState=OPENING, regionLocation=2998b885bdef,44417,1731482005920 2024-11-13T07:13:42,515 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=8ee3029b3e33cc034b84b2d370d7f61d, regionState=OPENING, regionLocation=2998b885bdef,37717,1731482006937 2024-11-13T07:13:42,515 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c5c88a6d20592c1b4adcc2269ffce351, regionState=OPENING, regionLocation=2998b885bdef,44417,1731482005920 2024-11-13T07:13:42,516 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=6af43b5a45829fff6c10e65235eba1d3, regionState=OPENING, regionLocation=2998b885bdef,37717,1731482006937 2024-11-13T07:13:42,516 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=5f4b94b50abb029a06bde24412d357dd, regionState=OPENING, regionLocation=2998b885bdef,37717,1731482006937 2024-11-13T07:13:42,516 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=feb86c9c548b3a71cf3eef83fbb2adeb, regionState=OPENING, regionLocation=2998b885bdef,36075,1731482006274 2024-11-13T07:13:42,516 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=f70ed29b296742bfa400816128951d24, regionState=OPENING, regionLocation=2998b885bdef,36075,1731482006274 2024-11-13T07:13:42,517 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=88b9c24c43eef7986e237d795658427e, regionState=OPENING, regionLocation=2998b885bdef,36075,1731482006274 2024-11-13T07:13:42,520 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=c880039bd4ef8e05f593aab556d82223, ASSIGN because future has completed 2024-11-13T07:13:42,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure c880039bd4ef8e05f593aab556d82223, server=2998b885bdef,44417,1731482005920}] 2024-11-13T07:13:42,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=8ee3029b3e33cc034b84b2d370d7f61d, ASSIGN because future has completed 2024-11-13T07:13:42,524 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ee3029b3e33cc034b84b2d370d7f61d, server=2998b885bdef,37717,1731482006937}] 2024-11-13T07:13:42,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=4b6c6542ce1ea1c1d3d96b4831e7c74a, ASSIGN because future has completed 2024-11-13T07:13:42,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=c5c88a6d20592c1b4adcc2269ffce351, ASSIGN because future has completed 2024-11-13T07:13:42,528 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c5c88a6d20592c1b4adcc2269ffce351, server=2998b885bdef,44417,1731482005920}] 2024-11-13T07:13:42,529 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=afca0ae186274ac99f7499d163ba2c4e, ASSIGN because future has completed 2024-11-13T07:13:42,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure afca0ae186274ac99f7499d163ba2c4e, server=2998b885bdef,44417,1731482005920}] 2024-11-13T07:13:42,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=5f4b94b50abb029a06bde24412d357dd, ASSIGN because future has completed 2024-11-13T07:13:42,532 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b6c6542ce1ea1c1d3d96b4831e7c74a, server=2998b885bdef,44417,1731482005920}] 2024-11-13T07:13:42,534 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=6af43b5a45829fff6c10e65235eba1d3, ASSIGN because future has completed 2024-11-13T07:13:42,537 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=feb86c9c548b3a71cf3eef83fbb2adeb, ASSIGN because future has completed 2024-11-13T07:13:42,541 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=f70ed29b296742bfa400816128951d24, ASSIGN because future has completed 2024-11-13T07:13:42,542 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure f70ed29b296742bfa400816128951d24, server=2998b885bdef,36075,1731482006274}] 2024-11-13T07:13:42,544 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=88b9c24c43eef7986e237d795658427e, ASSIGN because future has completed 2024-11-13T07:13:42,546 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure feb86c9c548b3a71cf3eef83fbb2adeb, server=2998b885bdef,36075,1731482006274}] 2024-11-13T07:13:42,546 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6af43b5a45829fff6c10e65235eba1d3, server=2998b885bdef,37717,1731482006937}] 2024-11-13T07:13:42,551 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f4b94b50abb029a06bde24412d357dd, server=2998b885bdef,37717,1731482006937}] 2024-11-13T07:13:42,551 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88b9c24c43eef7986e237d795658427e, server=2998b885bdef,36075,1731482006274}] 2024-11-13T07:13:42,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T07:13:44,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T07:13:45,160 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T07:13:45,160 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-13T07:13:45,164 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T07:13:45,164 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-13T07:13:46,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T07:13:46,875 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T07:13:50,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T07:13:52,527 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T07:13:52,529 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35265, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T07:13:52,533 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T07:13:52,535 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42955, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T07:13:52,551 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open SPTestTable,66666664,1731482021829.c880039bd4ef8e05f593aab556d82223. 2024-11-13T07:13:52,551 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => c880039bd4ef8e05f593aab556d82223, NAME => 'SPTestTable,66666664,1731482021829.c880039bd4ef8e05f593aab556d82223.', STARTKEY => '66666664', ENDKEY => '7ffffffd'} 2024-11-13T07:13:52,552 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table SPTestTable c880039bd4ef8e05f593aab556d82223 2024-11-13T07:13:52,552 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated SPTestTable,66666664,1731482021829.c880039bd4ef8e05f593aab556d82223.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:13:52,552 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for c880039bd4ef8e05f593aab556d82223 2024-11-13T07:13:52,552 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for c880039bd4ef8e05f593aab556d82223 2024-11-13T07:13:52,557 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(132): Open SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d. 2024-11-13T07:13:52,558 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7752): Opening region: {ENCODED => 8ee3029b3e33cc034b84b2d370d7f61d, NAME => 'SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d.', STARTKEY => 'ccccccc8', ENDKEY => 'e6666661'} 2024-11-13T07:13:52,558 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table SPTestTable 8ee3029b3e33cc034b84b2d370d7f61d 2024-11-13T07:13:52,559 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(898): Instantiated SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:13:52,559 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7794): checking encryption for 8ee3029b3e33cc034b84b2d370d7f61d 2024-11-13T07:13:52,559 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7797): checking classloading for 8ee3029b3e33cc034b84b2d370d7f61d 2024-11-13T07:13:52,560 INFO [StoreOpener-c880039bd4ef8e05f593aab556d82223-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region c880039bd4ef8e05f593aab556d82223 2024-11-13T07:13:52,564 INFO [StoreOpener-8ee3029b3e33cc034b84b2d370d7f61d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region 8ee3029b3e33cc034b84b2d370d7f61d 2024-11-13T07:13:52,570 INFO [StoreOpener-c880039bd4ef8e05f593aab556d82223-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c880039bd4ef8e05f593aab556d82223 columnFamilyName cf 2024-11-13T07:13:52,570 DEBUG [StoreOpener-c880039bd4ef8e05f593aab556d82223-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:13:52,571 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(132): Open SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24. 2024-11-13T07:13:52,571 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7752): Opening region: {ENCODED => f70ed29b296742bfa400816128951d24, NAME => 'SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24.', STARTKEY => '99999996', ENDKEY => 'b333332f'} 2024-11-13T07:13:52,571 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table SPTestTable f70ed29b296742bfa400816128951d24 2024-11-13T07:13:52,571 INFO [StoreOpener-8ee3029b3e33cc034b84b2d370d7f61d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ee3029b3e33cc034b84b2d370d7f61d columnFamilyName cf 2024-11-13T07:13:52,571 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(898): Instantiated SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:13:52,572 DEBUG [StoreOpener-8ee3029b3e33cc034b84b2d370d7f61d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:13:52,572 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7794): checking encryption for f70ed29b296742bfa400816128951d24 2024-11-13T07:13:52,572 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7797): checking classloading for f70ed29b296742bfa400816128951d24 2024-11-13T07:13:52,573 INFO [StoreOpener-c880039bd4ef8e05f593aab556d82223-1 {}] regionserver.HStore(327): Store=c880039bd4ef8e05f593aab556d82223/cf, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:13:52,573 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for c880039bd4ef8e05f593aab556d82223 2024-11-13T07:13:52,575 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/c880039bd4ef8e05f593aab556d82223 2024-11-13T07:13:52,575 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/c880039bd4ef8e05f593aab556d82223 2024-11-13T07:13:52,576 INFO [StoreOpener-8ee3029b3e33cc034b84b2d370d7f61d-1 {}] regionserver.HStore(327): Store=8ee3029b3e33cc034b84b2d370d7f61d/cf, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:13:52,576 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for c880039bd4ef8e05f593aab556d82223 2024-11-13T07:13:52,576 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for c880039bd4ef8e05f593aab556d82223 2024-11-13T07:13:52,577 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1038): replaying wal for 8ee3029b3e33cc034b84b2d370d7f61d 2024-11-13T07:13:52,578 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/8ee3029b3e33cc034b84b2d370d7f61d 2024-11-13T07:13:52,579 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/8ee3029b3e33cc034b84b2d370d7f61d 2024-11-13T07:13:52,581 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1048): stopping wal replay for 8ee3029b3e33cc034b84b2d370d7f61d 2024-11-13T07:13:52,581 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for c880039bd4ef8e05f593aab556d82223 2024-11-13T07:13:52,581 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1060): Cleaning up temporary data for 8ee3029b3e33cc034b84b2d370d7f61d 2024-11-13T07:13:52,582 INFO [StoreOpener-f70ed29b296742bfa400816128951d24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region f70ed29b296742bfa400816128951d24 2024-11-13T07:13:52,584 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1093): writing seq id for 8ee3029b3e33cc034b84b2d370d7f61d 2024-11-13T07:13:52,586 INFO [StoreOpener-f70ed29b296742bfa400816128951d24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f70ed29b296742bfa400816128951d24 columnFamilyName cf 2024-11-13T07:13:52,586 DEBUG [StoreOpener-f70ed29b296742bfa400816128951d24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:13:52,586 INFO [StoreOpener-f70ed29b296742bfa400816128951d24-1 {}] regionserver.HStore(327): Store=f70ed29b296742bfa400816128951d24/cf, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:13:52,587 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1038): replaying wal for f70ed29b296742bfa400816128951d24 2024-11-13T07:13:52,588 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/f70ed29b296742bfa400816128951d24 2024-11-13T07:13:52,590 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/f70ed29b296742bfa400816128951d24 2024-11-13T07:13:52,591 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1048): stopping wal replay for f70ed29b296742bfa400816128951d24 2024-11-13T07:13:52,591 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1060): Cleaning up temporary data for f70ed29b296742bfa400816128951d24 2024-11-13T07:13:52,592 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/c880039bd4ef8e05f593aab556d82223/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:13:52,599 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1093): writing seq id for f70ed29b296742bfa400816128951d24 2024-11-13T07:13:52,599 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened c880039bd4ef8e05f593aab556d82223; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66872634, jitterRate=-0.003520101308822632}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:13:52,600 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c880039bd4ef8e05f593aab556d82223 2024-11-13T07:13:52,601 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for c880039bd4ef8e05f593aab556d82223: Running coprocessor pre-open hook at 1731482032552Writing region info on filesystem at 1731482032552Initializing all the Stores at 1731482032556 (+4 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482032556Cleaning up temporary data from old regions at 1731482032576 (+20 ms)Running coprocessor post-open hooks at 1731482032600 (+24 ms)Region opened successfully at 1731482032601 (+1 ms) 2024-11-13T07:13:52,604 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/8ee3029b3e33cc034b84b2d370d7f61d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:13:52,608 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/f70ed29b296742bfa400816128951d24/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:13:52,611 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for SPTestTable,66666664,1731482021829.c880039bd4ef8e05f593aab556d82223., pid=15, masterSystemTime=1731482032526 2024-11-13T07:13:52,611 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1114): Opened 8ee3029b3e33cc034b84b2d370d7f61d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60357166, jitterRate=-0.10060814023017883}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:13:52,612 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8ee3029b3e33cc034b84b2d370d7f61d 2024-11-13T07:13:52,612 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1006): Region open journal for 8ee3029b3e33cc034b84b2d370d7f61d: Running coprocessor pre-open hook at 1731482032559Writing region info on filesystem at 1731482032559Initializing all the Stores at 1731482032562 (+3 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482032562Cleaning up temporary data from old regions at 1731482032581 (+19 ms)Running coprocessor post-open hooks at 1731482032612 (+31 ms)Region opened successfully at 1731482032612 2024-11-13T07:13:52,612 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1114): Opened f70ed29b296742bfa400816128951d24; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68575529, jitterRate=0.021855011582374573}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:13:52,612 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f70ed29b296742bfa400816128951d24 2024-11-13T07:13:52,613 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1006): Region open journal for f70ed29b296742bfa400816128951d24: Running coprocessor pre-open hook at 1731482032572Writing region info on filesystem at 1731482032572Initializing all the Stores at 1731482032580 (+8 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482032580Cleaning up temporary data from old regions at 1731482032591 (+11 ms)Running coprocessor post-open hooks at 1731482032612 (+21 ms)Region opened successfully at 1731482032613 (+1 ms) 2024-11-13T07:13:52,738 INFO [AsyncFSWAL-0-hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/MasterData-prefix:2998b885bdef,46239,1731482004610 {}] wal.AbstractFSWAL(1368): Slow sync cost: 122 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:40257,DS-14e25fb3-d25b-4488-aa49-d869ee0f7b89,DISK], DatanodeInfoWithStorage[127.0.0.1:38559,DS-f0b0f387-c446-4c2b-8ab1-3fda3b8c8206,DISK], DatanodeInfoWithStorage[127.0.0.1:38027,DS-77acad21-8279-4d7b-afc4-61162fa08ba0,DISK]] 2024-11-13T07:13:52,739 INFO [AsyncFSWAL-0-hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9-prefix:2998b885bdef,37717,1731482006937 {}] wal.AbstractFSWAL(1368): Slow sync cost: 126 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38559,DS-f0b0f387-c446-4c2b-8ab1-3fda3b8c8206,DISK], DatanodeInfoWithStorage[127.0.0.1:38027,DS-77acad21-8279-4d7b-afc4-61162fa08ba0,DISK], DatanodeInfoWithStorage[127.0.0.1:40257,DS-14e25fb3-d25b-4488-aa49-d869ee0f7b89,DISK]] 2024-11-13T07:13:52,739 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2236): Post open deploy tasks for SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d., pid=16, masterSystemTime=1731482032533 2024-11-13T07:13:52,739 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for SPTestTable,66666664,1731482021829.c880039bd4ef8e05f593aab556d82223. 2024-11-13T07:13:52,739 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened SPTestTable,66666664,1731482021829.c880039bd4ef8e05f593aab556d82223. 2024-11-13T07:13:52,739 INFO [AsyncFSWAL-0-hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9-prefix:2998b885bdef,36075,1731482006274 {}] wal.AbstractFSWAL(1368): Slow sync cost: 125 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:38559,DS-f0b0f387-c446-4c2b-8ab1-3fda3b8c8206,DISK], DatanodeInfoWithStorage[127.0.0.1:38027,DS-77acad21-8279-4d7b-afc4-61162fa08ba0,DISK], DatanodeInfoWithStorage[127.0.0.1:40257,DS-14e25fb3-d25b-4488-aa49-d869ee0f7b89,DISK]] 2024-11-13T07:13:52,739 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open SPTestTable,,1731482021829.c5c88a6d20592c1b4adcc2269ffce351. 2024-11-13T07:13:52,739 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => c5c88a6d20592c1b4adcc2269ffce351, NAME => 'SPTestTable,,1731482021829.c5c88a6d20592c1b4adcc2269ffce351.', STARTKEY => '', ENDKEY => '19999999'} 2024-11-13T07:13:52,739 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2236): Post open deploy tasks for SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24., pid=20, masterSystemTime=1731482032548 2024-11-13T07:13:52,740 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table SPTestTable c5c88a6d20592c1b4adcc2269ffce351 2024-11-13T07:13:52,740 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated SPTestTable,,1731482021829.c5c88a6d20592c1b4adcc2269ffce351.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:13:52,740 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for c5c88a6d20592c1b4adcc2269ffce351 2024-11-13T07:13:52,740 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for c5c88a6d20592c1b4adcc2269ffce351 2024-11-13T07:13:52,742 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=c880039bd4ef8e05f593aab556d82223, regionState=OPEN, openSeqNum=2, regionLocation=2998b885bdef,44417,1731482005920 2024-11-13T07:13:52,745 INFO [StoreOpener-c5c88a6d20592c1b4adcc2269ffce351-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region c5c88a6d20592c1b4adcc2269ffce351 2024-11-13T07:13:52,748 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2266): Finished post open deploy task for SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24. 2024-11-13T07:13:52,748 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(153): Opened SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24. 2024-11-13T07:13:52,748 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=f70ed29b296742bfa400816128951d24, regionState=OPEN, openSeqNum=2, regionLocation=2998b885bdef,36075,1731482006274 2024-11-13T07:13:52,748 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e. 2024-11-13T07:13:52,749 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 88b9c24c43eef7986e237d795658427e, NAME => 'SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e.', STARTKEY => '4ccccccb', ENDKEY => '66666664'} 2024-11-13T07:13:52,749 INFO [StoreOpener-c5c88a6d20592c1b4adcc2269ffce351-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5c88a6d20592c1b4adcc2269ffce351 columnFamilyName cf 2024-11-13T07:13:52,749 DEBUG [StoreOpener-c5c88a6d20592c1b4adcc2269ffce351-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:13:52,750 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table SPTestTable 88b9c24c43eef7986e237d795658427e 2024-11-13T07:13:52,750 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:13:52,750 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2266): Finished post open deploy task for SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d. 2024-11-13T07:13:52,750 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(153): Opened SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d. 2024-11-13T07:13:52,750 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] handler.AssignRegionHandler(132): Open SPTestTable,7ffffffd,1731482021829.6af43b5a45829fff6c10e65235eba1d3. 2024-11-13T07:13:52,751 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(7752): Opening region: {ENCODED => 6af43b5a45829fff6c10e65235eba1d3, NAME => 'SPTestTable,7ffffffd,1731482021829.6af43b5a45829fff6c10e65235eba1d3.', STARTKEY => '7ffffffd', ENDKEY => '99999996'} 2024-11-13T07:13:52,751 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 88b9c24c43eef7986e237d795658427e 2024-11-13T07:13:52,751 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 88b9c24c43eef7986e237d795658427e 2024-11-13T07:13:52,751 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table SPTestTable 6af43b5a45829fff6c10e65235eba1d3 2024-11-13T07:13:52,751 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=8ee3029b3e33cc034b84b2d370d7f61d, regionState=OPEN, openSeqNum=2, regionLocation=2998b885bdef,37717,1731482006937 2024-11-13T07:13:52,751 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(898): Instantiated SPTestTable,7ffffffd,1731482021829.6af43b5a45829fff6c10e65235eba1d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:13:52,751 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(7794): checking encryption for 6af43b5a45829fff6c10e65235eba1d3 2024-11-13T07:13:52,751 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(7797): checking classloading for 6af43b5a45829fff6c10e65235eba1d3 2024-11-13T07:13:52,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure c880039bd4ef8e05f593aab556d82223, server=2998b885bdef,44417,1731482005920 because future has completed 2024-11-13T07:13:52,754 INFO [StoreOpener-c5c88a6d20592c1b4adcc2269ffce351-1 {}] regionserver.HStore(327): Store=c5c88a6d20592c1b4adcc2269ffce351/cf, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:13:52,754 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for c5c88a6d20592c1b4adcc2269ffce351 2024-11-13T07:13:52,755 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/c5c88a6d20592c1b4adcc2269ffce351 2024-11-13T07:13:52,756 INFO [StoreOpener-88b9c24c43eef7986e237d795658427e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region 88b9c24c43eef7986e237d795658427e 2024-11-13T07:13:52,757 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/c5c88a6d20592c1b4adcc2269ffce351 2024-11-13T07:13:52,758 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for c5c88a6d20592c1b4adcc2269ffce351 2024-11-13T07:13:52,758 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for c5c88a6d20592c1b4adcc2269ffce351 2024-11-13T07:13:52,758 INFO [StoreOpener-88b9c24c43eef7986e237d795658427e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88b9c24c43eef7986e237d795658427e columnFamilyName cf 2024-11-13T07:13:52,759 DEBUG [StoreOpener-88b9c24c43eef7986e237d795658427e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:13:52,759 INFO [StoreOpener-88b9c24c43eef7986e237d795658427e-1 {}] regionserver.HStore(327): Store=88b9c24c43eef7986e237d795658427e/cf, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:13:52,760 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 88b9c24c43eef7986e237d795658427e 2024-11-13T07:13:52,761 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/88b9c24c43eef7986e237d795658427e 2024-11-13T07:13:52,762 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/88b9c24c43eef7986e237d795658427e 2024-11-13T07:13:52,762 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 88b9c24c43eef7986e237d795658427e 2024-11-13T07:13:52,762 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 88b9c24c43eef7986e237d795658427e 2024-11-13T07:13:52,764 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for c5c88a6d20592c1b4adcc2269ffce351 2024-11-13T07:13:52,767 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=9 2024-11-13T07:13:52,768 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure c880039bd4ef8e05f593aab556d82223, server=2998b885bdef,44417,1731482005920 in 10.2400 sec 2024-11-13T07:13:52,768 INFO [StoreOpener-6af43b5a45829fff6c10e65235eba1d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region 6af43b5a45829fff6c10e65235eba1d3 2024-11-13T07:13:52,769 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure f70ed29b296742bfa400816128951d24, server=2998b885bdef,36075,1731482006274 because future has completed 2024-11-13T07:13:52,769 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ee3029b3e33cc034b84b2d370d7f61d, server=2998b885bdef,37717,1731482006937 because future has completed 2024-11-13T07:13:52,775 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 88b9c24c43eef7986e237d795658427e 2024-11-13T07:13:52,776 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=c880039bd4ef8e05f593aab556d82223, ASSIGN in 10.4130 sec 2024-11-13T07:13:52,776 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/c5c88a6d20592c1b4adcc2269ffce351/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:13:52,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-13T07:13:52,779 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 8ee3029b3e33cc034b84b2d370d7f61d, server=2998b885bdef,37717,1731482006937 in 10.2490 sec 2024-11-13T07:13:52,780 INFO [StoreOpener-6af43b5a45829fff6c10e65235eba1d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6af43b5a45829fff6c10e65235eba1d3 columnFamilyName cf 2024-11-13T07:13:52,780 DEBUG [StoreOpener-6af43b5a45829fff6c10e65235eba1d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:13:52,782 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened c5c88a6d20592c1b4adcc2269ffce351; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64318902, jitterRate=-0.041573673486709595}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:13:52,782 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c5c88a6d20592c1b4adcc2269ffce351 2024-11-13T07:13:52,782 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for c5c88a6d20592c1b4adcc2269ffce351: Running coprocessor pre-open hook at 1731482032740Writing region info on filesystem at 1731482032740Initializing all the Stores at 1731482032745 (+5 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482032745Cleaning up temporary data from old regions at 1731482032758 (+13 ms)Running coprocessor post-open hooks at 1731482032782 (+24 ms)Region opened successfully at 1731482032782 2024-11-13T07:13:52,784 INFO [StoreOpener-6af43b5a45829fff6c10e65235eba1d3-1 {}] regionserver.HStore(327): Store=6af43b5a45829fff6c10e65235eba1d3/cf, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:13:52,784 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1038): replaying wal for 6af43b5a45829fff6c10e65235eba1d3 2024-11-13T07:13:52,786 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/6af43b5a45829fff6c10e65235eba1d3 2024-11-13T07:13:52,787 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/88b9c24c43eef7986e237d795658427e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:13:52,787 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/6af43b5a45829fff6c10e65235eba1d3 2024-11-13T07:13:52,788 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for SPTestTable,,1731482021829.c5c88a6d20592c1b4adcc2269ffce351., pid=17, masterSystemTime=1731482032526 2024-11-13T07:13:52,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=11 2024-11-13T07:13:52,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure f70ed29b296742bfa400816128951d24, server=2998b885bdef,36075,1731482006274 in 10.2310 sec 2024-11-13T07:13:52,791 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1048): stopping wal replay for 6af43b5a45829fff6c10e65235eba1d3 2024-11-13T07:13:52,791 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1060): Cleaning up temporary data for 6af43b5a45829fff6c10e65235eba1d3 2024-11-13T07:13:52,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=8ee3029b3e33cc034b84b2d370d7f61d, ASSIGN in 10.4250 sec 2024-11-13T07:13:52,790 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 88b9c24c43eef7986e237d795658427e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68199317, jitterRate=0.01624901592731476}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:13:52,791 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 88b9c24c43eef7986e237d795658427e 2024-11-13T07:13:52,792 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for SPTestTable,,1731482021829.c5c88a6d20592c1b4adcc2269ffce351. 2024-11-13T07:13:52,792 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened SPTestTable,,1731482021829.c5c88a6d20592c1b4adcc2269ffce351. 2024-11-13T07:13:52,792 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e. 2024-11-13T07:13:52,793 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => afca0ae186274ac99f7499d163ba2c4e, NAME => 'SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e.', STARTKEY => 'b333332f', ENDKEY => 'ccccccc8'} 2024-11-13T07:13:52,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=f70ed29b296742bfa400816128951d24, ASSIGN in 10.4340 sec 2024-11-13T07:13:52,793 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c5c88a6d20592c1b4adcc2269ffce351, regionState=OPEN, openSeqNum=2, regionLocation=2998b885bdef,44417,1731482005920 2024-11-13T07:13:52,794 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table SPTestTable afca0ae186274ac99f7499d163ba2c4e 2024-11-13T07:13:52,795 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 88b9c24c43eef7986e237d795658427e: Running coprocessor pre-open hook at 1731482032751Writing region info on filesystem at 1731482032751Initializing all the Stores at 1731482032755 (+4 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482032755Cleaning up temporary data from old regions at 1731482032763 (+8 ms)Running coprocessor post-open hooks at 1731482032791 (+28 ms)Region opened successfully at 1731482032791 2024-11-13T07:13:52,794 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:13:52,797 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for afca0ae186274ac99f7499d163ba2c4e 2024-11-13T07:13:52,797 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for afca0ae186274ac99f7499d163ba2c4e 2024-11-13T07:13:52,800 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1093): writing seq id for 6af43b5a45829fff6c10e65235eba1d3 2024-11-13T07:13:52,805 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/6af43b5a45829fff6c10e65235eba1d3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:13:52,807 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1114): Opened 6af43b5a45829fff6c10e65235eba1d3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71878382, jitterRate=0.07107135653495789}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:13:52,807 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6af43b5a45829fff6c10e65235eba1d3 2024-11-13T07:13:52,807 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1006): Region open journal for 6af43b5a45829fff6c10e65235eba1d3: Running coprocessor pre-open hook at 1731482032752Writing region info on filesystem at 1731482032752Initializing all the Stores at 1731482032763 (+11 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482032763Cleaning up temporary data from old regions at 1731482032791 (+28 ms)Running coprocessor post-open hooks at 1731482032807 (+16 ms)Region opened successfully at 1731482032807 2024-11-13T07:13:52,808 INFO [StoreOpener-afca0ae186274ac99f7499d163ba2c4e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region afca0ae186274ac99f7499d163ba2c4e 2024-11-13T07:13:52,809 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegionServer(2236): Post open deploy tasks for SPTestTable,7ffffffd,1731482021829.6af43b5a45829fff6c10e65235eba1d3., pid=22, masterSystemTime=1731482032533 2024-11-13T07:13:52,810 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e., pid=24, masterSystemTime=1731482032548 2024-11-13T07:13:52,813 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c5c88a6d20592c1b4adcc2269ffce351, server=2998b885bdef,44417,1731482005920 because future has completed 2024-11-13T07:13:52,814 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegionServer(2266): Finished post open deploy task for SPTestTable,7ffffffd,1731482021829.6af43b5a45829fff6c10e65235eba1d3. 2024-11-13T07:13:52,814 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] handler.AssignRegionHandler(153): Opened SPTestTable,7ffffffd,1731482021829.6af43b5a45829fff6c10e65235eba1d3. 2024-11-13T07:13:52,814 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(132): Open SPTestTable,19999999,1731482021829.5f4b94b50abb029a06bde24412d357dd. 2024-11-13T07:13:52,815 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7752): Opening region: {ENCODED => 5f4b94b50abb029a06bde24412d357dd, NAME => 'SPTestTable,19999999,1731482021829.5f4b94b50abb029a06bde24412d357dd.', STARTKEY => '19999999', ENDKEY => '33333332'} 2024-11-13T07:13:52,815 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table SPTestTable 5f4b94b50abb029a06bde24412d357dd 2024-11-13T07:13:52,815 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(898): Instantiated SPTestTable,19999999,1731482021829.5f4b94b50abb029a06bde24412d357dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:13:52,816 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7794): checking encryption for 5f4b94b50abb029a06bde24412d357dd 2024-11-13T07:13:52,816 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7797): checking classloading for 5f4b94b50abb029a06bde24412d357dd 2024-11-13T07:13:52,817 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=6af43b5a45829fff6c10e65235eba1d3, regionState=OPEN, openSeqNum=2, regionLocation=2998b885bdef,37717,1731482006937 2024-11-13T07:13:52,821 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e. 2024-11-13T07:13:52,821 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e. 2024-11-13T07:13:52,821 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb. 2024-11-13T07:13:52,821 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => feb86c9c548b3a71cf3eef83fbb2adeb, NAME => 'SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb.', STARTKEY => 'e6666661', ENDKEY => ''} 2024-11-13T07:13:52,822 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table SPTestTable feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:13:52,822 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:13:52,822 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:13:52,823 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=88b9c24c43eef7986e237d795658427e, regionState=OPEN, openSeqNum=2, regionLocation=2998b885bdef,36075,1731482006274 2024-11-13T07:13:52,824 INFO [StoreOpener-5f4b94b50abb029a06bde24412d357dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region 5f4b94b50abb029a06bde24412d357dd 2024-11-13T07:13:52,822 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:13:52,824 INFO [StoreOpener-afca0ae186274ac99f7499d163ba2c4e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region afca0ae186274ac99f7499d163ba2c4e columnFamilyName cf 2024-11-13T07:13:52,828 DEBUG [StoreOpener-afca0ae186274ac99f7499d163ba2c4e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:13:52,830 INFO [StoreOpener-feb86c9c548b3a71cf3eef83fbb2adeb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:13:52,830 INFO [StoreOpener-5f4b94b50abb029a06bde24412d357dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f4b94b50abb029a06bde24412d357dd columnFamilyName cf 2024-11-13T07:13:52,832 INFO [StoreOpener-afca0ae186274ac99f7499d163ba2c4e-1 {}] regionserver.HStore(327): Store=afca0ae186274ac99f7499d163ba2c4e/cf, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:13:52,833 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for afca0ae186274ac99f7499d163ba2c4e 2024-11-13T07:13:52,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6af43b5a45829fff6c10e65235eba1d3, server=2998b885bdef,37717,1731482006937 because future has completed 2024-11-13T07:13:52,834 INFO [StoreOpener-feb86c9c548b3a71cf3eef83fbb2adeb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region feb86c9c548b3a71cf3eef83fbb2adeb columnFamilyName cf 2024-11-13T07:13:52,834 DEBUG [StoreOpener-feb86c9c548b3a71cf3eef83fbb2adeb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:13:52,835 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/afca0ae186274ac99f7499d163ba2c4e 2024-11-13T07:13:52,837 INFO [StoreOpener-feb86c9c548b3a71cf3eef83fbb2adeb-1 {}] regionserver.HStore(327): Store=feb86c9c548b3a71cf3eef83fbb2adeb/cf, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:13:52,838 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/afca0ae186274ac99f7499d163ba2c4e 2024-11-13T07:13:52,838 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:13:52,838 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for afca0ae186274ac99f7499d163ba2c4e 2024-11-13T07:13:52,839 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for afca0ae186274ac99f7499d163ba2c4e 2024-11-13T07:13:52,840 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:13:52,841 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:13:52,842 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=5 2024-11-13T07:13:52,843 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46239 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=2998b885bdef,36075,1731482006274, table=SPTestTable, region=88b9c24c43eef7986e237d795658427e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T07:13:52,844 DEBUG [StoreOpener-5f4b94b50abb029a06bde24412d357dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:13:52,846 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:13:52,846 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:13:52,849 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:13:52,858 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c5c88a6d20592c1b4adcc2269ffce351, server=2998b885bdef,44417,1731482005920 in 10.2940 sec 2024-11-13T07:13:52,858 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for afca0ae186274ac99f7499d163ba2c4e 2024-11-13T07:13:52,859 INFO [StoreOpener-5f4b94b50abb029a06bde24412d357dd-1 {}] regionserver.HStore(327): Store=5f4b94b50abb029a06bde24412d357dd/cf, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:13:52,859 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1038): replaying wal for 5f4b94b50abb029a06bde24412d357dd 2024-11-13T07:13:52,860 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/5f4b94b50abb029a06bde24412d357dd 2024-11-13T07:13:52,861 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/5f4b94b50abb029a06bde24412d357dd 2024-11-13T07:13:52,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88b9c24c43eef7986e237d795658427e, server=2998b885bdef,36075,1731482006274 because future has completed 2024-11-13T07:13:52,861 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=c5c88a6d20592c1b4adcc2269ffce351, ASSIGN in 10.4900 sec 2024-11-13T07:13:52,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=10 2024-11-13T07:13:52,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 6af43b5a45829fff6c10e65235eba1d3, server=2998b885bdef,37717,1731482006937 in 10.2940 sec 2024-11-13T07:13:52,862 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1048): stopping wal replay for 5f4b94b50abb029a06bde24412d357dd 2024-11-13T07:13:52,862 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1060): Cleaning up temporary data for 5f4b94b50abb029a06bde24412d357dd 2024-11-13T07:13:52,867 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1093): writing seq id for 5f4b94b50abb029a06bde24412d357dd 2024-11-13T07:13:52,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=6af43b5a45829fff6c10e65235eba1d3, ASSIGN in 10.5070 sec 2024-11-13T07:13:52,878 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/feb86c9c548b3a71cf3eef83fbb2adeb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:13:52,879 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened feb86c9c548b3a71cf3eef83fbb2adeb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60618233, jitterRate=-0.0967179387807846}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:13:52,879 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:13:52,879 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for feb86c9c548b3a71cf3eef83fbb2adeb: Running coprocessor pre-open hook at 1731482032826Writing region info on filesystem at 1731482032826Initializing all the Stores at 1731482032829 (+3 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482032829Cleaning up temporary data from old regions at 1731482032846 (+17 ms)Running coprocessor post-open hooks at 1731482032879 (+33 ms)Region opened successfully at 1731482032879 2024-11-13T07:13:52,882 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/afca0ae186274ac99f7499d163ba2c4e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:13:52,883 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened afca0ae186274ac99f7499d163ba2c4e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66146863, jitterRate=-0.01433493196964264}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:13:52,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=8 2024-11-13T07:13:52,883 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for afca0ae186274ac99f7499d163ba2c4e 2024-11-13T07:13:52,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 88b9c24c43eef7986e237d795658427e, server=2998b885bdef,36075,1731482006274 in 10.3140 sec 2024-11-13T07:13:52,883 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for afca0ae186274ac99f7499d163ba2c4e: Running coprocessor pre-open hook at 1731482032798Writing region info on filesystem at 1731482032798Initializing all the Stores at 1731482032801 (+3 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482032801Cleaning up temporary data from old regions at 1731482032839 (+38 ms)Running coprocessor post-open hooks at 1731482032883 (+44 ms)Region opened successfully at 1731482032883 2024-11-13T07:13:52,884 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/5f4b94b50abb029a06bde24412d357dd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:13:52,884 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb., pid=21, masterSystemTime=1731482032548 2024-11-13T07:13:52,888 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1114): Opened 5f4b94b50abb029a06bde24412d357dd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72660403, jitterRate=0.08272437751293182}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:13:52,888 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=88b9c24c43eef7986e237d795658427e, ASSIGN in 10.5280 sec 2024-11-13T07:13:52,888 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5f4b94b50abb029a06bde24412d357dd 2024-11-13T07:13:52,888 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e., pid=18, masterSystemTime=1731482032526 2024-11-13T07:13:52,888 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1006): Region open journal for 5f4b94b50abb029a06bde24412d357dd: Running coprocessor pre-open hook at 1731482032816Writing region info on filesystem at 1731482032816Initializing all the Stores at 1731482032820 (+4 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482032820Cleaning up temporary data from old regions at 1731482032862 (+42 ms)Running coprocessor post-open hooks at 1731482032888 (+26 ms)Region opened successfully at 1731482032888 2024-11-13T07:13:52,889 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb. 2024-11-13T07:13:52,889 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb. 2024-11-13T07:13:52,892 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=feb86c9c548b3a71cf3eef83fbb2adeb, regionState=OPEN, openSeqNum=2, regionLocation=2998b885bdef,36075,1731482006274 2024-11-13T07:13:52,896 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2236): Post open deploy tasks for SPTestTable,19999999,1731482021829.5f4b94b50abb029a06bde24412d357dd., pid=23, masterSystemTime=1731482032533 2024-11-13T07:13:52,904 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e. 2024-11-13T07:13:52,904 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e. 2024-11-13T07:13:52,904 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(132): Open SPTestTable,33333332,1731482021829.4b6c6542ce1ea1c1d3d96b4831e7c74a. 2024-11-13T07:13:52,904 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7752): Opening region: {ENCODED => 4b6c6542ce1ea1c1d3d96b4831e7c74a, NAME => 'SPTestTable,33333332,1731482021829.4b6c6542ce1ea1c1d3d96b4831e7c74a.', STARTKEY => '33333332', ENDKEY => '4ccccccb'} 2024-11-13T07:13:52,905 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table SPTestTable 4b6c6542ce1ea1c1d3d96b4831e7c74a 2024-11-13T07:13:52,905 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(898): Instantiated SPTestTable,33333332,1731482021829.4b6c6542ce1ea1c1d3d96b4831e7c74a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:13:52,905 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7794): checking encryption for 4b6c6542ce1ea1c1d3d96b4831e7c74a 2024-11-13T07:13:52,905 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7797): checking classloading for 4b6c6542ce1ea1c1d3d96b4831e7c74a 2024-11-13T07:13:52,914 INFO [StoreOpener-4b6c6542ce1ea1c1d3d96b4831e7c74a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region 4b6c6542ce1ea1c1d3d96b4831e7c74a 2024-11-13T07:13:52,915 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=afca0ae186274ac99f7499d163ba2c4e, regionState=OPEN, openSeqNum=2, regionLocation=2998b885bdef,44417,1731482005920 2024-11-13T07:13:52,917 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2266): Finished post open deploy task for SPTestTable,19999999,1731482021829.5f4b94b50abb029a06bde24412d357dd. 2024-11-13T07:13:52,917 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(153): Opened SPTestTable,19999999,1731482021829.5f4b94b50abb029a06bde24412d357dd. 2024-11-13T07:13:52,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure feb86c9c548b3a71cf3eef83fbb2adeb, server=2998b885bdef,36075,1731482006274 because future has completed 2024-11-13T07:13:52,923 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=5f4b94b50abb029a06bde24412d357dd, regionState=OPEN, openSeqNum=2, regionLocation=2998b885bdef,37717,1731482006937 2024-11-13T07:13:52,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure afca0ae186274ac99f7499d163ba2c4e, server=2998b885bdef,44417,1731482005920 because future has completed 2024-11-13T07:13:52,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f4b94b50abb029a06bde24412d357dd, server=2998b885bdef,37717,1731482006937 because future has completed 2024-11-13T07:13:52,933 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=14 2024-11-13T07:13:52,937 INFO [StoreOpener-4b6c6542ce1ea1c1d3d96b4831e7c74a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b6c6542ce1ea1c1d3d96b4831e7c74a columnFamilyName cf 2024-11-13T07:13:52,937 DEBUG [StoreOpener-4b6c6542ce1ea1c1d3d96b4831e7c74a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:13:52,938 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure feb86c9c548b3a71cf3eef83fbb2adeb, server=2998b885bdef,36075,1731482006274 in 10.3800 sec 2024-11-13T07:13:52,941 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=12 2024-11-13T07:13:52,943 INFO [StoreOpener-4b6c6542ce1ea1c1d3d96b4831e7c74a-1 {}] regionserver.HStore(327): Store=4b6c6542ce1ea1c1d3d96b4831e7c74a/cf, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:13:52,943 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1038): replaying wal for 4b6c6542ce1ea1c1d3d96b4831e7c74a 2024-11-13T07:13:52,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=6 2024-11-13T07:13:52,944 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure afca0ae186274ac99f7499d163ba2c4e, server=2998b885bdef,44417,1731482005920 in 10.4030 sec 2024-11-13T07:13:52,946 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/4b6c6542ce1ea1c1d3d96b4831e7c74a 2024-11-13T07:13:52,947 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 5f4b94b50abb029a06bde24412d357dd, server=2998b885bdef,37717,1731482006937 in 10.3830 sec 2024-11-13T07:13:52,947 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=feb86c9c548b3a71cf3eef83fbb2adeb, ASSIGN in 10.5840 sec 2024-11-13T07:13:52,948 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/4b6c6542ce1ea1c1d3d96b4831e7c74a 2024-11-13T07:13:52,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=afca0ae186274ac99f7499d163ba2c4e, ASSIGN in 10.5870 sec 2024-11-13T07:13:52,950 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=5f4b94b50abb029a06bde24412d357dd, ASSIGN in 10.5890 sec 2024-11-13T07:13:52,952 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1048): stopping wal replay for 4b6c6542ce1ea1c1d3d96b4831e7c74a 2024-11-13T07:13:52,953 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1060): Cleaning up temporary data for 4b6c6542ce1ea1c1d3d96b4831e7c74a 2024-11-13T07:13:52,956 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1093): writing seq id for 4b6c6542ce1ea1c1d3d96b4831e7c74a 2024-11-13T07:13:52,968 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/4b6c6542ce1ea1c1d3d96b4831e7c74a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:13:52,969 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1114): Opened 4b6c6542ce1ea1c1d3d96b4831e7c74a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74503510, jitterRate=0.1101888120174408}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:13:52,970 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4b6c6542ce1ea1c1d3d96b4831e7c74a 2024-11-13T07:13:52,970 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1006): Region open journal for 4b6c6542ce1ea1c1d3d96b4831e7c74a: Running coprocessor pre-open hook at 1731482032905Writing region info on filesystem at 1731482032905Initializing all the Stores at 1731482032913 (+8 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482032913Cleaning up temporary data from old regions at 1731482032953 (+40 ms)Running coprocessor post-open hooks at 1731482032970 (+17 ms)Region opened successfully at 1731482032970 2024-11-13T07:13:52,972 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2236): Post open deploy tasks for SPTestTable,33333332,1731482021829.4b6c6542ce1ea1c1d3d96b4831e7c74a., pid=19, masterSystemTime=1731482032526 2024-11-13T07:13:52,975 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2266): Finished post open deploy task for SPTestTable,33333332,1731482021829.4b6c6542ce1ea1c1d3d96b4831e7c74a. 2024-11-13T07:13:52,975 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(153): Opened SPTestTable,33333332,1731482021829.4b6c6542ce1ea1c1d3d96b4831e7c74a. 2024-11-13T07:13:52,977 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=4b6c6542ce1ea1c1d3d96b4831e7c74a, regionState=OPEN, openSeqNum=2, regionLocation=2998b885bdef,44417,1731482005920 2024-11-13T07:13:52,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b6c6542ce1ea1c1d3d96b4831e7c74a, server=2998b885bdef,44417,1731482005920 because future has completed 2024-11-13T07:13:52,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=7 2024-11-13T07:13:52,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 4b6c6542ce1ea1c1d3d96b4831e7c74a, server=2998b885bdef,44417,1731482005920 in 10.4600 sec 2024-11-13T07:13:53,006 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=7, resume processing ppid=4 2024-11-13T07:13:53,007 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=4b6c6542ce1ea1c1d3d96b4831e7c74a, ASSIGN in 10.6440 sec 2024-11-13T07:13:53,009 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=SPTestTable execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T07:13:53,010 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"SPTestTable","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731482033009"}]},"ts":"1731482033009"} 2024-11-13T07:13:53,016 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=SPTestTable, state=ENABLED in hbase:meta 2024-11-13T07:13:53,022 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=SPTestTable execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T07:13:53,029 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=SPTestTable in 11.1840 sec 2024-11-13T07:13:53,296 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T07:13:54,170 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'SPTestTable' 2024-11-13T07:13:55,160 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_SPTestTable 2024-11-13T07:13:55,160 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_SPTestTable Metrics about Tables on a single HBase RegionServer 2024-11-13T07:13:58,126 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T07:14:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T07:14:00,163 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:SPTestTable completed 2024-11-13T07:14:00,163 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table SPTestTable get assigned. Timeout = 60000ms 2024-11-13T07:14:00,164 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T07:14:00,181 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table SPTestTable assigned to meta. Checking AM states. 2024-11-13T07:14:00,181 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T07:14:00,182 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table SPTestTable assigned. 2024-11-13T07:14:00,279 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='aab', locateType=CURRENT is [region=SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24., hostname=2998b885bdef,36075,1731482006274, seqNum=2] 2024-11-13T07:14:00,287 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='aaa', locateType=CURRENT is [region=SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24., hostname=2998b885bdef,36075,1731482006274, seqNum=2] 2024-11-13T07:14:00,299 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='aac', locateType=CURRENT is [region=SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24., hostname=2998b885bdef,36075,1731482006274, seqNum=2] 2024-11-13T07:14:00,302 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='aae', locateType=CURRENT is [region=SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24., hostname=2998b885bdef,36075,1731482006274, seqNum=2] 2024-11-13T07:14:00,307 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='aad', locateType=CURRENT is [region=SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24., hostname=2998b885bdef,36075,1731482006274, seqNum=2] 2024-11-13T07:14:00,316 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='aag', locateType=CURRENT is [region=SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24., hostname=2998b885bdef,36075,1731482006274, seqNum=2] 2024-11-13T07:14:00,329 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='aah', locateType=CURRENT is [region=SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24., hostname=2998b885bdef,36075,1731482006274, seqNum=2] 2024-11-13T07:14:00,332 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='aaf', locateType=CURRENT is [region=SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24., hostname=2998b885bdef,36075,1731482006274, seqNum=2] 2024-11-13T07:14:00,449 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='bab', locateType=CURRENT is [region=SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e., hostname=2998b885bdef,44417,1731482005920, seqNum=2] 2024-11-13T07:14:00,461 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='baa', locateType=CURRENT is [region=SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e., hostname=2998b885bdef,44417,1731482005920, seqNum=2] 2024-11-13T07:14:00,478 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='bac', locateType=CURRENT is [region=SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e., hostname=2998b885bdef,44417,1731482005920, seqNum=2] 2024-11-13T07:14:00,487 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='bad', locateType=CURRENT is [region=SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e., hostname=2998b885bdef,44417,1731482005920, seqNum=2] 2024-11-13T07:14:00,490 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='bae', locateType=CURRENT is [region=SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e., hostname=2998b885bdef,44417,1731482005920, seqNum=2] 2024-11-13T07:14:00,519 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='bag', locateType=CURRENT is [region=SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e., hostname=2998b885bdef,44417,1731482005920, seqNum=2] 2024-11-13T07:14:00,522 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='baf', locateType=CURRENT is [region=SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e., hostname=2998b885bdef,44417,1731482005920, seqNum=2] 2024-11-13T07:14:00,546 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='bah', locateType=CURRENT is [region=SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e., hostname=2998b885bdef,44417,1731482005920, seqNum=2] 2024-11-13T07:14:00,555 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='cck', locateType=CURRENT is [region=SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d., hostname=2998b885bdef,37717,1731482006937, seqNum=2] 2024-11-13T07:14:00,558 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='ccd', locateType=CURRENT is [region=SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d., hostname=2998b885bdef,37717,1731482006937, seqNum=2] 2024-11-13T07:14:00,569 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='ccf', locateType=CURRENT is [region=SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d., hostname=2998b885bdef,37717,1731482006937, seqNum=2] 2024-11-13T07:14:00,571 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='ccg', locateType=CURRENT is [region=SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d., hostname=2998b885bdef,37717,1731482006937, seqNum=2] 2024-11-13T07:14:00,578 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='cch', locateType=CURRENT is [region=SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d., hostname=2998b885bdef,37717,1731482006937, seqNum=2] 2024-11-13T07:14:00,582 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='cce', locateType=CURRENT is [region=SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d., hostname=2998b885bdef,37717,1731482006937, seqNum=2] 2024-11-13T07:14:00,597 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='ccj', locateType=CURRENT is [region=SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d., hostname=2998b885bdef,37717,1731482006937, seqNum=2] 2024-11-13T07:14:00,599 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='cci', locateType=CURRENT is [region=SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d., hostname=2998b885bdef,37717,1731482006937, seqNum=2] 2024-11-13T07:14:00,696 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='eaa', locateType=CURRENT is [region=SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb., hostname=2998b885bdef,36075,1731482006274, seqNum=2] 2024-11-13T07:14:00,704 DEBUG [Time-limited test {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='eab', locateType=CURRENT is [region=SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb., hostname=2998b885bdef,36075,1731482006274, seqNum=2] 2024-11-13T07:14:02,546 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T07:14:02,587 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T07:14:02,654 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T07:14:02,670 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T07:14:02,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36075 {}] regionserver.RSRpcServices(2610): Large batch operation detected (greater than 5000) (HBASE-18023). Requested Number of Rows: 15548 Client: jenkins//172.17.0.3 first region in multi=SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24. 2024-11-13T07:14:02,851 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44417 {}] regionserver.HRegion(8528): writing data to region SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e. with WAL disabled. Data may be lost in the event of a crash. 2024-11-13T07:14:02,916 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36075 {}] regionserver.HRegion(8528): writing data to region SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24. with WAL disabled. Data may be lost in the event of a crash. 2024-11-13T07:14:02,933 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37717 {}] regionserver.HRegion(8528): writing data to region SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d. with WAL disabled. Data may be lost in the event of a crash. 2024-11-13T07:14:04,035 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36075 {}] regionserver.HRegion(8528): writing data to region SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb. with WAL disabled. Data may be lost in the event of a crash. 2024-11-13T07:14:04,788 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=25, snapshot={ ss=SnapshotProcedureTest table=SPTestTable type=FLUSH ttl=0 } 2024-11-13T07:14:04,794 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=25, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=25, snapshot={ ss=SnapshotProcedureTest table=SPTestTable type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-13T07:14:04,802 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=25, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=25, snapshot={ ss=SnapshotProcedureTest table=SPTestTable type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-13T07:14:04,805 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T07:14:04,853 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=25, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=25, snapshot={ ss=SnapshotProcedureTest table=SPTestTable type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-13T07:14:04,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741848_1024 (size=49) 2024-11-13T07:14:04,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741848_1024 (size=49) 2024-11-13T07:14:04,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741848_1024 (size=49) 2024-11-13T07:14:04,965 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=25, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=25, snapshot={ ss=SnapshotProcedureTest table=SPTestTable type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-13T07:14:04,968 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure c5c88a6d20592c1b4adcc2269ffce351}, {pid=27, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 5f4b94b50abb029a06bde24412d357dd}, {pid=28, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 4b6c6542ce1ea1c1d3d96b4831e7c74a}, {pid=29, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 88b9c24c43eef7986e237d795658427e}, {pid=30, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure c880039bd4ef8e05f593aab556d82223}, {pid=31, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 6af43b5a45829fff6c10e65235eba1d3}, {pid=32, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f70ed29b296742bfa400816128951d24}, {pid=33, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure afca0ae186274ac99f7499d163ba2c4e}, {pid=34, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 8ee3029b3e33cc034b84b2d370d7f61d}, {pid=35, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure feb86c9c548b3a71cf3eef83fbb2adeb}] 2024-11-13T07:14:04,995 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure c5c88a6d20592c1b4adcc2269ffce351 2024-11-13T07:14:04,997 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 5f4b94b50abb029a06bde24412d357dd 2024-11-13T07:14:05,005 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure c880039bd4ef8e05f593aab556d82223 2024-11-13T07:14:05,005 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure afca0ae186274ac99f7499d163ba2c4e 2024-11-13T07:14:05,006 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 8ee3029b3e33cc034b84b2d370d7f61d 2024-11-13T07:14:05,007 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f70ed29b296742bfa400816128951d24 2024-11-13T07:14:05,007 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:14:05,009 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 2998b885bdef,36075,1731482006274 2024-11-13T07:14:05,010 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 2998b885bdef,36075,1731482006274: Simulated kill ***** 2024-11-13T07:14:05,010 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-13T07:14:05,013 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-13T07:14:05,015 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-13T07:14:05,019 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-13T07:14:05,020 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-13T07:14:05,023 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 6af43b5a45829fff6c10e65235eba1d3 2024-11-13T07:14:05,023 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 88b9c24c43eef7986e237d795658427e 2024-11-13T07:14:05,027 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 4b6c6542ce1ea1c1d3d96b4831e7c74a 2024-11-13T07:14:05,061 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 381280768 }, "NonHeapMemoryUsage": { "committed": 157220864, "init": 7667712, "max": -1, "used": 153674136 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "2998b885bdef", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 1, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 14486, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 35, "ProcessCallTime_min": 0, "ProcessCallTime_max": 0, "ProcessCallTime_mean": 0, "ProcessCallTime_25th_percentile": 0, "ProcessCallTime_median": 0, "ProcessCallTime_75th_percentile": 0, "ProcessCallTime_90th_percentile": 0, "ProcessCallTime_95th_percentile": 0, "ProcessCallTime_98th_percentile": 0, "ProcessCallTime_99th_percentile": 0, "ProcessCallTime_99.9th_percentile": 0, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 35, "QueueCallTime_min": 0, "QueueCallTime_max": 0, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 35, "TotalCallTime_min": 0, "TotalCallTime_max": 0, "TotalCallTime_mean": 0, "TotalCallTime_25th_percentile": 0, "TotalCallTime_median": 0, "TotalCallTime_75th_percentile": 0, "TotalCallTime_90th_percentile": 0, "TotalCallTime_95th_percentile": 0, "TotalCallTime_98th_percentile": 0, "TotalCallTime_99th_percentile": 0, "TotalCallTime_99.9th_percentile": 0, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 35, "ResponseSize_min": 0, "ResponseSize_max": 0, "ResponseSize_mean": 0, "ResponseSize_25th_percentile": 0, "ResponseSize_median": 0, "ResponseSize_75th_percentile": 0, "ResponseSize_90th_percentile": 0, "ResponseSize_95th_percentile": 0, "ResponseSize_98th_percentile": 0, "ResponseSize_99th_percentile": 0, "ResponseSize_99.9th_percentile": 0, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 35, "RequestSize_min": 0, "RequestSize_max": 0, "RequestSize_mean": 0, "RequestSize_25th_percentile": 0, "RequestSize_median": 0, "RequestSize_75th_percentile": 0, "RequestSize_90th_percentile": 0, "RequestSize_95th_percentile": 0, "RequestSize_98th_percentile": 0, "RequestSize_99th_percentile": 0, "RequestSize_99.9th_percentile": 0, "sentBytes": 676 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "2998b885bdef", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:56395", "tag.serverName": "2998b885bdef,36075,1731482006274", "tag.clusterId": "05a88896-60ba-480c-907e-788ba4493ab2", "tag.Context": "regionserver", "tag.Hostname": "2998b885bdef", "regionCount": 4, "storeCount": 7, "hlogFileCount": 8, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 12732, "memStoreHeapSize": 22368, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731482006274, "averageRegionSize": 3183, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 0, "localBytesRead": 0, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 2.2, "ByteBuffAllocatorHeapAllocationBytes": 0, "ByteBuffAllocatorPoolAllocationBytes": 266240, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 31, "totalRowActionRequestCount": 42, "readRequestCount": 8, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 34, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 4, "rpcMultiRequestCount": 1, "rpcMutateRequestCount": 24, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 4, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 24, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 1, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 4, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-13T07:14:05,068 WARN [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46239 {}] master.MasterRpcServices(700): 2998b885bdef,36075,1731482006274 reported a fatal error: ***** ABORTING region server 2998b885bdef,36075,1731482006274: Simulated kill ***** 2024-11-13T07:14:05,085 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2998b885bdef,36075,1731482006274' ***** 2024-11-13T07:14:05,085 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-13T07:14:05,085 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T07:14:05,087 INFO [RS:1;2998b885bdef:36075 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T07:14:05,087 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T07:14:05,087 INFO [RS:1;2998b885bdef:36075 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-13T07:14:05,088 INFO [RS:1;2998b885bdef:36075 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-13T07:14:05,088 INFO [RS:1;2998b885bdef:36075 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T07:14:05,088 INFO [RS:1;2998b885bdef:36075 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;2998b885bdef:36075. 2024-11-13T07:14:05,088 DEBUG [RS:1;2998b885bdef:36075 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T07:14:05,088 DEBUG [RS:1;2998b885bdef:36075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:14:05,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741836_1012 (size=14382) 2024-11-13T07:14:05,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741836_1012 (size=14382) 2024-11-13T07:14:05,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741836_1012 (size=14382) 2024-11-13T07:14:05,152 INFO [regionserver/2998b885bdef:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T07:14:05,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741833_1009 (size=966) 2024-11-13T07:14:05,190 DEBUG [RS:1;2998b885bdef:36075 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:14:05,190 INFO [RS:1;2998b885bdef:36075 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T07:14:05,191 INFO [RS:1;2998b885bdef:36075 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36075 2024-11-13T07:14:05,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741833_1009 (size=966) 2024-11-13T07:14:05,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741833_1009 (size=966) 2024-11-13T07:14:05,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36075-0x101330aa3af0002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2998b885bdef,36075,1731482006274 2024-11-13T07:14:05,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T07:14:05,264 INFO [RS:1;2998b885bdef:36075 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T07:14:05,278 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2998b885bdef,36075,1731482006274] 2024-11-13T07:14:05,295 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2998b885bdef,36075,1731482006274 already deleted, retry=false 2024-11-13T07:14:05,296 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 2998b885bdef,36075,1731482006274 on 2998b885bdef,46239,1731482004610 2024-11-13T07:14:05,303 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 2998b885bdef,36075,1731482006274, splitWal=true, meta=true 2024-11-13T07:14:05,309 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=36 for 2998b885bdef,36075,1731482006274 (carryingMeta=true) 2998b885bdef,36075,1731482006274/CRASHED/regionCount=4/lock=java.util.concurrent.locks.ReentrantReadWriteLock@460b9c72[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-13T07:14:05,313 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=36, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 2998b885bdef,36075,1731482006274, splitWal=true, meta=true 2024-11-13T07:14:05,318 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=36, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 2998b885bdef,36075,1731482006274, splitWal=true, meta=true, isMeta: true 2024-11-13T07:14:05,324 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:14:05,327 WARN [PEWorker-4 {}] procedure.SnapshotRegionProcedure(211): pid=35, ppid=25, state=RUNNABLE, hasLock=true; SnapshotRegionProcedure feb86c9c548b3a71cf3eef83fbb2adeb can not run currently because target server of region SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb. 2998b885bdef,36075,1731482006274 is in state SPLITTING_META, wait 1000 ms to retry 2024-11-13T07:14:05,327 INFO [PEWorker-4 {}] procedure2.TimeoutExecutorThread(82): ADDED pid=35, ppid=25, state=WAITING_TIMEOUT, hasLock=true; SnapshotRegionProcedure feb86c9c548b3a71cf3eef83fbb2adeb; timeout=1000, timestamp=1731482046327 2024-11-13T07:14:05,329 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274-splitting 2024-11-13T07:14:05,332 INFO [PEWorker-2 {}] master.SplitWALManager(105): 2998b885bdef,36075,1731482006274 WAL count=1, meta=true 2024-11-13T07:14:05,335 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 2998b885bdef%2C36075%2C1731482006274.meta.1731482021283.meta}] 2024-11-13T07:14:05,342 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 88b9c24c43eef7986e237d795658427e 2024-11-13T07:14:05,343 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f70ed29b296742bfa400816128951d24 2024-11-13T07:14:05,344 WARN [PEWorker-5 {}] procedure.SnapshotRegionProcedure(211): pid=29, ppid=25, state=RUNNABLE, hasLock=true; SnapshotRegionProcedure 88b9c24c43eef7986e237d795658427e can not run currently because target server of region SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e. 2998b885bdef,36075,1731482006274 is in state SPLITTING_META, wait 1000 ms to retry 2024-11-13T07:14:05,344 INFO [PEWorker-5 {}] procedure2.TimeoutExecutorThread(82): ADDED pid=29, ppid=25, state=WAITING_TIMEOUT, hasLock=true; SnapshotRegionProcedure 88b9c24c43eef7986e237d795658427e; timeout=1000, timestamp=1731482046344 2024-11-13T07:14:05,345 WARN [PEWorker-3 {}] procedure.SnapshotRegionProcedure(211): pid=32, ppid=25, state=RUNNABLE, hasLock=true; SnapshotRegionProcedure f70ed29b296742bfa400816128951d24 can not run currently because target server of region SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24. 2998b885bdef,36075,1731482006274 is in state SPLITTING_META, wait 1000 ms to retry 2024-11-13T07:14:05,345 INFO [PEWorker-3 {}] procedure2.TimeoutExecutorThread(82): ADDED pid=32, ppid=25, state=WAITING_TIMEOUT, hasLock=true; SnapshotRegionProcedure f70ed29b296742bfa400816128951d24; timeout=1000, timestamp=1731482046345 2024-11-13T07:14:05,347 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=2998b885bdef,44417,1731482005920 2024-11-13T07:14:05,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 2998b885bdef%2C36075%2C1731482006274.meta.1731482021283.meta, worker=2998b885bdef,44417,1731482005920}] 2024-11-13T07:14:05,379 INFO [RS:1;2998b885bdef:36075 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T07:14:05,380 INFO [RS:1;2998b885bdef:36075 {}] regionserver.HRegionServer(1031): Exiting; stopping=2998b885bdef,36075,1731482006274; zookeeper connection closed. 2024-11-13T07:14:05,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36075-0x101330aa3af0002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T07:14:05,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36075-0x101330aa3af0002, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T07:14:05,383 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@67812e9c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@67812e9c 2024-11-13T07:14:05,456 INFO [regionserver/2998b885bdef:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T07:14:05,456 INFO [regionserver/2998b885bdef:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T07:14:06,327 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:14:06,336 WARN [PEWorker-2 {}] procedure.SnapshotRegionProcedure(211): pid=35, ppid=25, state=RUNNABLE, hasLock=true; SnapshotRegionProcedure feb86c9c548b3a71cf3eef83fbb2adeb can not run currently because target server of region SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb. 2998b885bdef,36075,1731482006274 is in state SPLITTING_META, wait 2000 ms to retry 2024-11-13T07:14:06,336 INFO [PEWorker-2 {}] procedure2.TimeoutExecutorThread(82): ADDED pid=35, ppid=25, state=WAITING_TIMEOUT, hasLock=true; SnapshotRegionProcedure feb86c9c548b3a71cf3eef83fbb2adeb; timeout=2000, timestamp=1731482048336 2024-11-13T07:14:06,344 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 88b9c24c43eef7986e237d795658427e 2024-11-13T07:14:06,346 WARN [PEWorker-5 {}] procedure.SnapshotRegionProcedure(211): pid=29, ppid=25, state=RUNNABLE, hasLock=true; SnapshotRegionProcedure 88b9c24c43eef7986e237d795658427e can not run currently because target server of region SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e. 2998b885bdef,36075,1731482006274 is in state SPLITTING_META, wait 2000 ms to retry 2024-11-13T07:14:06,346 INFO [PEWorker-5 {}] procedure2.TimeoutExecutorThread(82): ADDED pid=29, ppid=25, state=WAITING_TIMEOUT, hasLock=true; SnapshotRegionProcedure 88b9c24c43eef7986e237d795658427e; timeout=2000, timestamp=1731482048346 2024-11-13T07:14:06,347 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f70ed29b296742bfa400816128951d24 2024-11-13T07:14:06,349 WARN [PEWorker-3 {}] procedure.SnapshotRegionProcedure(211): pid=32, ppid=25, state=RUNNABLE, hasLock=true; SnapshotRegionProcedure f70ed29b296742bfa400816128951d24 can not run currently because target server of region SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24. 2998b885bdef,36075,1731482006274 is in state SPLITTING_META, wait 2000 ms to retry 2024-11-13T07:14:06,349 INFO [PEWorker-3 {}] procedure2.TimeoutExecutorThread(82): ADDED pid=32, ppid=25, state=WAITING_TIMEOUT, hasLock=true; SnapshotRegionProcedure f70ed29b296742bfa400816128951d24; timeout=2000, timestamp=1731482048349 2024-11-13T07:14:08,337 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:14:08,339 WARN [PEWorker-4 {}] procedure.SnapshotRegionProcedure(211): pid=35, ppid=25, state=RUNNABLE, hasLock=true; SnapshotRegionProcedure feb86c9c548b3a71cf3eef83fbb2adeb can not run currently because target server of region SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb. 2998b885bdef,36075,1731482006274 is in state SPLITTING_META, wait 4000 ms to retry 2024-11-13T07:14:08,339 INFO [PEWorker-4 {}] procedure2.TimeoutExecutorThread(82): ADDED pid=35, ppid=25, state=WAITING_TIMEOUT, hasLock=true; SnapshotRegionProcedure feb86c9c548b3a71cf3eef83fbb2adeb; timeout=4000, timestamp=1731482052339 2024-11-13T07:14:08,346 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 88b9c24c43eef7986e237d795658427e 2024-11-13T07:14:08,347 WARN [PEWorker-1 {}] procedure.SnapshotRegionProcedure(211): pid=29, ppid=25, state=RUNNABLE, hasLock=true; SnapshotRegionProcedure 88b9c24c43eef7986e237d795658427e can not run currently because target server of region SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e. 2998b885bdef,36075,1731482006274 is in state SPLITTING_META, wait 4000 ms to retry 2024-11-13T07:14:08,347 INFO [PEWorker-1 {}] procedure2.TimeoutExecutorThread(82): ADDED pid=29, ppid=25, state=WAITING_TIMEOUT, hasLock=true; SnapshotRegionProcedure 88b9c24c43eef7986e237d795658427e; timeout=4000, timestamp=1731482052347 2024-11-13T07:14:08,349 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f70ed29b296742bfa400816128951d24 2024-11-13T07:14:08,350 WARN [PEWorker-2 {}] procedure.SnapshotRegionProcedure(211): pid=32, ppid=25, state=RUNNABLE, hasLock=true; SnapshotRegionProcedure f70ed29b296742bfa400816128951d24 can not run currently because target server of region SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24. 2998b885bdef,36075,1731482006274 is in state SPLITTING_META, wait 4000 ms to retry 2024-11-13T07:14:08,350 INFO [PEWorker-2 {}] procedure2.TimeoutExecutorThread(82): ADDED pid=32, ppid=25, state=WAITING_TIMEOUT, hasLock=true; SnapshotRegionProcedure f70ed29b296742bfa400816128951d24; timeout=4000, timestamp=1731482052350 2024-11-13T07:14:12,339 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:14:12,343 WARN [PEWorker-5 {}] procedure.SnapshotRegionProcedure(211): pid=35, ppid=25, state=RUNNABLE, hasLock=true; SnapshotRegionProcedure feb86c9c548b3a71cf3eef83fbb2adeb can not run currently because target server of region SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb. 2998b885bdef,36075,1731482006274 is in state SPLITTING_META, wait 8000 ms to retry 2024-11-13T07:14:12,343 INFO [PEWorker-5 {}] procedure2.TimeoutExecutorThread(82): ADDED pid=35, ppid=25, state=WAITING_TIMEOUT, hasLock=true; SnapshotRegionProcedure feb86c9c548b3a71cf3eef83fbb2adeb; timeout=8000, timestamp=1731482060343 2024-11-13T07:14:12,348 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 88b9c24c43eef7986e237d795658427e 2024-11-13T07:14:12,351 WARN [PEWorker-5 {}] procedure.SnapshotRegionProcedure(211): pid=29, ppid=25, state=RUNNABLE, hasLock=true; SnapshotRegionProcedure 88b9c24c43eef7986e237d795658427e can not run currently because target server of region SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e. 2998b885bdef,36075,1731482006274 is in state SPLITTING_META, wait 8000 ms to retry 2024-11-13T07:14:12,351 INFO [PEWorker-5 {}] procedure2.TimeoutExecutorThread(82): ADDED pid=29, ppid=25, state=WAITING_TIMEOUT, hasLock=true; SnapshotRegionProcedure 88b9c24c43eef7986e237d795658427e; timeout=8000, timestamp=1731482060351 2024-11-13T07:14:12,357 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f70ed29b296742bfa400816128951d24 2024-11-13T07:14:12,361 WARN [PEWorker-4 {}] procedure.SnapshotRegionProcedure(211): pid=32, ppid=25, state=RUNNABLE, hasLock=true; SnapshotRegionProcedure f70ed29b296742bfa400816128951d24 can not run currently because target server of region SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24. 2998b885bdef,36075,1731482006274 is in state SPLITTING_META, wait 8000 ms to retry 2024-11-13T07:14:12,361 INFO [PEWorker-4 {}] procedure2.TimeoutExecutorThread(82): ADDED pid=32, ppid=25, state=WAITING_TIMEOUT, hasLock=true; SnapshotRegionProcedure f70ed29b296742bfa400816128951d24; timeout=8000, timestamp=1731482060361 2024-11-13T07:14:12,548 ERROR [ProcedureDispatcherTimeoutThread {}] procedure2.RemoteProcedureDispatcher$TimeoutExecutorThread(331): DelayQueue for RemoteProcedureDispatcher is not empty when timed waiting elapsed. If this is repeated consistently, it means no element is getting expired from the queue and it might freeze the system. Queue: [containedObject=2998b885bdef,44417,1731482005920, timeout=1731482054999, delay=2450, operations=[pid=30, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure c880039bd4ef8e05f593aab556d82223, pid=33, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure afca0ae186274ac99f7499d163ba2c4e, pid=28, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 4b6c6542ce1ea1c1d3d96b4831e7c74a, pid=38, ppid=37, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 2998b885bdef%2C36075%2C1731482006274.meta.1731482021283.meta, worker=2998b885bdef,44417,1731482005920, pid=26, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure c5c88a6d20592c1b4adcc2269ffce351], containedObject=2998b885bdef,37717,1731482006937, timeout=1731482055000, delay=2451, operations=[pid=27, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 5f4b94b50abb029a06bde24412d357dd, pid=34, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 8ee3029b3e33cc034b84b2d370d7f61d, pid=31, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 6af43b5a45829fff6c10e65235eba1d3], containedObject=2998b885bdef,36075,1731482006274, timeout=1731482055009, delay=2460, operations=null] 2024-11-13T07:14:15,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=27 2024-11-13T07:14:15,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=30 2024-11-13T07:14:15,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2024-11-13T07:14:15,005 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=30}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on SPTestTable,66666664,1731482021829.c880039bd4ef8e05f593aab556d82223. 2024-11-13T07:14:15,006 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=27}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on SPTestTable,19999999,1731482021829.5f4b94b50abb029a06bde24412d357dd. 2024-11-13T07:14:15,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=28 2024-11-13T07:14:15,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=34 2024-11-13T07:14:15,006 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e. 2024-11-13T07:14:15,006 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=28}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on SPTestTable,33333332,1731482021829.4b6c6542ce1ea1c1d3d96b4831e7c74a. 2024-11-13T07:14:15,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=38 2024-11-13T07:14:15,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=31 2024-11-13T07:14:15,007 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=34}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d. 2024-11-13T07:14:15,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=26 2024-11-13T07:14:15,008 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=31}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on SPTestTable,7ffffffd,1731482021829.6af43b5a45829fff6c10e65235eba1d3. 2024-11-13T07:14:15,011 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2902): Flushing afca0ae186274ac99f7499d163ba2c4e 1/1 column families, dataSize=21.42 KB heapSize=74.49 KB 2024-11-13T07:14:15,011 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 4b6c6542ce1ea1c1d3d96b4831e7c74a: 2024-11-13T07:14:15,011 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=28}] regionserver.SnapshotRegionCallable(75): Snapshotting region SPTestTable,33333332,1731482021829.4b6c6542ce1ea1c1d3d96b4831e7c74a. for SnapshotProcedureTest completed. 2024-11-13T07:14:15,011 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=27}] regionserver.HRegion(2603): Flush status journal for 5f4b94b50abb029a06bde24412d357dd: 2024-11-13T07:14:15,012 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=27}] regionserver.SnapshotRegionCallable(75): Snapshotting region SPTestTable,19999999,1731482021829.5f4b94b50abb029a06bde24412d357dd. for SnapshotProcedureTest completed. 2024-11-13T07:14:15,012 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=31}] regionserver.HRegion(2603): Flush status journal for 6af43b5a45829fff6c10e65235eba1d3: 2024-11-13T07:14:15,012 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=31}] regionserver.SnapshotRegionCallable(75): Snapshotting region SPTestTable,7ffffffd,1731482021829.6af43b5a45829fff6c10e65235eba1d3. for SnapshotProcedureTest completed. 2024-11-13T07:14:15,012 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=27}] snapshot.SnapshotManifest(241): Storing 'SPTestTable,19999999,1731482021829.5f4b94b50abb029a06bde24412d357dd.' region-info for snapshot=SnapshotProcedureTest 2024-11-13T07:14:15,013 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=31}] snapshot.SnapshotManifest(241): Storing 'SPTestTable,7ffffffd,1731482021829.6af43b5a45829fff6c10e65235eba1d3.' region-info for snapshot=SnapshotProcedureTest 2024-11-13T07:14:15,013 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=28}] snapshot.SnapshotManifest(241): Storing 'SPTestTable,33333332,1731482021829.4b6c6542ce1ea1c1d3d96b4831e7c74a.' region-info for snapshot=SnapshotProcedureTest 2024-11-13T07:14:15,014 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=30}] regionserver.HRegion(2603): Flush status journal for c880039bd4ef8e05f593aab556d82223: 2024-11-13T07:14:15,014 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=30}] regionserver.SnapshotRegionCallable(75): Snapshotting region SPTestTable,66666664,1731482021829.c880039bd4ef8e05f593aab556d82223. for SnapshotProcedureTest completed. 2024-11-13T07:14:15,014 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=30}] snapshot.SnapshotManifest(241): Storing 'SPTestTable,66666664,1731482021829.c880039bd4ef8e05f593aab556d82223.' region-info for snapshot=SnapshotProcedureTest 2024-11-13T07:14:15,011 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=34}] regionserver.HRegion(2902): Flushing 8ee3029b3e33cc034b84b2d370d7f61d 1/1 column families, dataSize=38.00 KB heapSize=131.98 KB 2024-11-13T07:14:15,016 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=27}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-13T07:14:15,016 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=30}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-13T07:14:15,016 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=31}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-13T07:14:15,016 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=28}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-13T07:14:15,019 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=27}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-13T07:14:15,019 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=31}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-13T07:14:15,019 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=30}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-13T07:14:15,020 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=28}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-13T07:14:15,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741850_1026 (size=65) 2024-11-13T07:14:15,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741850_1026 (size=65) 2024-11-13T07:14:15,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741850_1026 (size=65) 2024-11-13T07:14:15,042 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=28}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on SPTestTable,33333332,1731482021829.4b6c6542ce1ea1c1d3d96b4831e7c74a. 2024-11-13T07:14:15,044 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-13T07:14:15,045 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=38}] wal.WALSplitter(299): Splitting hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274-splitting/2998b885bdef%2C36075%2C1731482006274.meta.1731482021283.meta, size=14.0 K (14382bytes) 2024-11-13T07:14:15,045 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=26}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on SPTestTable,,1731482021829.c5c88a6d20592c1b4adcc2269ffce351. 2024-11-13T07:14:15,046 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for c5c88a6d20592c1b4adcc2269ffce351: 2024-11-13T07:14:15,046 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=26}] regionserver.SnapshotRegionCallable(75): Snapshotting region SPTestTable,,1731482021829.c5c88a6d20592c1b4adcc2269ffce351. for SnapshotProcedureTest completed. 2024-11-13T07:14:15,046 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=26}] snapshot.SnapshotManifest(241): Storing 'SPTestTable,,1731482021829.c5c88a6d20592c1b4adcc2269ffce351.' region-info for snapshot=SnapshotProcedureTest 2024-11-13T07:14:15,046 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=26}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-13T07:14:15,046 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=26}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-13T07:14:15,046 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=38}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-13T07:14:15,047 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=38}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-13T07:14:15,047 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=38}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274-splitting/2998b885bdef%2C36075%2C1731482006274.meta.1731482021283.meta 2024-11-13T07:14:15,053 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=38}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274-splitting/2998b885bdef%2C36075%2C1731482006274.meta.1731482021283.meta after 4ms 2024-11-13T07:14:15,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741849_1025 (size=65) 2024-11-13T07:14:15,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741849_1025 (size=65) 2024-11-13T07:14:15,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741849_1025 (size=65) 2024-11-13T07:14:15,057 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=30}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on SPTestTable,66666664,1731482021829.c880039bd4ef8e05f593aab556d82223. 2024-11-13T07:14:15,057 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-11-13T07:14:15,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-13T07:14:15,058 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] procedure.SnapshotRegionProcedure(132): finish snapshot SnapshotProcedureTest on region 4b6c6542ce1ea1c1d3d96b4831e7c74a 2024-11-13T07:14:15,059 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 4b6c6542ce1ea1c1d3d96b4831e7c74a 2024-11-13T07:14:15,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.HMaster(4169): Remote procedure done, pid=30 2024-11-13T07:14:15,060 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] procedure.SnapshotRegionProcedure(132): finish snapshot SnapshotProcedureTest on region c880039bd4ef8e05f593aab556d82223 2024-11-13T07:14:15,060 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure c880039bd4ef8e05f593aab556d82223 2024-11-13T07:14:15,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=25, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 4b6c6542ce1ea1c1d3d96b4831e7c74a in 10.0930 sec 2024-11-13T07:14:15,065 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=25, state=SUCCESS, hasLock=false; SnapshotRegionProcedure c880039bd4ef8e05f593aab556d82223 in 10.0940 sec 2024-11-13T07:14:15,093 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=38}] wal.WALSplitter(310): Open hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274-splitting/2998b885bdef%2C36075%2C1731482006274.meta.1731482021283.meta took 50ms 2024-11-13T07:14:15,139 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=38}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-13T07:14:15,148 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=38}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274-splitting/2998b885bdef%2C36075%2C1731482006274.meta.1731482021283.meta so closing down 2024-11-13T07:14:15,148 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=38}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T07:14:15,149 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-2998b885bdef%2C36075%2C1731482006274.meta.1731482021283.meta.temp 2024-11-13T07:14:15,160 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/afca0ae186274ac99f7499d163ba2c4e/.tmp/cf/b42be0ac08494614a648964fe6d9b775 is 34, key is baa/cf:cf/1731482042835/Put/seqid=0 2024-11-13T07:14:15,160 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/8ee3029b3e33cc034b84b2d370d7f61d/.tmp/cf/51fade2042e24b6ba4e2ca43c74b884f is 34, key is ccd/cf:cf/1731482042906/Put/seqid=0 2024-11-13T07:14:15,160 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2998b885bdef%2C36075%2C1731482006274.meta.1731482021283.meta.temp 2024-11-13T07:14:15,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741853_1029 (size=65) 2024-11-13T07:14:15,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741851_1027 (size=57) 2024-11-13T07:14:15,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741853_1029 (size=65) 2024-11-13T07:14:15,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741853_1029 (size=65) 2024-11-13T07:14:15,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741851_1027 (size=57) 2024-11-13T07:14:15,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741851_1027 (size=57) 2024-11-13T07:14:15,166 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=27}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on SPTestTable,19999999,1731482021829.5f4b94b50abb029a06bde24412d357dd. 2024-11-13T07:14:15,166 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=26}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on SPTestTable,,1731482021829.c5c88a6d20592c1b4adcc2269ffce351. 2024-11-13T07:14:15,166 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=27 2024-11-13T07:14:15,167 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-13T07:14:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46239 {}] master.HMaster(4169): Remote procedure done, pid=27 2024-11-13T07:14:15,168 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46239 {}] procedure.SnapshotRegionProcedure(132): finish snapshot SnapshotProcedureTest on region 5f4b94b50abb029a06bde24412d357dd 2024-11-13T07:14:15,168 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 5f4b94b50abb029a06bde24412d357dd 2024-11-13T07:14:15,168 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=38}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T07:14:15,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-13T07:14:15,169 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] procedure.SnapshotRegionProcedure(132): finish snapshot SnapshotProcedureTest on region c5c88a6d20592c1b4adcc2269ffce351 2024-11-13T07:14:15,169 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure c5c88a6d20592c1b4adcc2269ffce351 2024-11-13T07:14:15,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741852_1028 (size=65) 2024-11-13T07:14:15,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741852_1028 (size=65) 2024-11-13T07:14:15,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741852_1028 (size=65) 2024-11-13T07:14:15,186 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=31}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on SPTestTable,7ffffffd,1731482021829.6af43b5a45829fff6c10e65235eba1d3. 2024-11-13T07:14:15,186 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=31}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=31 2024-11-13T07:14:15,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.HMaster(4169): Remote procedure done, pid=31 2024-11-13T07:14:15,187 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] procedure.SnapshotRegionProcedure(132): finish snapshot SnapshotProcedureTest on region 6af43b5a45829fff6c10e65235eba1d3 2024-11-13T07:14:15,193 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 6af43b5a45829fff6c10e65235eba1d3 2024-11-13T07:14:15,193 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; SnapshotRegionProcedure c5c88a6d20592c1b4adcc2269ffce351 in 10.2020 sec 2024-11-13T07:14:15,195 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=25, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 5f4b94b50abb029a06bde24412d357dd in 10.2020 sec 2024-11-13T07:14:15,197 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=25, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 6af43b5a45829fff6c10e65235eba1d3 in 10.2260 sec 2024-11-13T07:14:15,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741855_1031 (size=47197) 2024-11-13T07:14:15,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741855_1031 (size=47197) 2024-11-13T07:14:15,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741855_1031 (size=47197) 2024-11-13T07:14:15,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741854_1030 (size=28623) 2024-11-13T07:14:15,239 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.00 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/8ee3029b3e33cc034b84b2d370d7f61d/.tmp/cf/51fade2042e24b6ba4e2ca43c74b884f 2024-11-13T07:14:15,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741854_1030 (size=28623) 2024-11-13T07:14:15,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741854_1030 (size=28623) 2024-11-13T07:14:15,240 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=21.42 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/afca0ae186274ac99f7499d163ba2c4e/.tmp/cf/b42be0ac08494614a648964fe6d9b775 2024-11-13T07:14:15,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741856_1032 (size=14141) 2024-11-13T07:14:15,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741856_1032 (size=14141) 2024-11-13T07:14:15,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741856_1032 (size=14141) 2024-11-13T07:14:15,264 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2998b885bdef%2C36075%2C1731482006274.meta.1731482021283.meta.temp (wrote 25 edits, skipped 0 edits in 2 ms) 2024-11-13T07:14:15,267 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2998b885bdef%2C36075%2C1731482006274.meta.1731482021283.meta.temp to hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/recovered.edits/0000000000000000028 2024-11-13T07:14:15,271 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=38}] wal.WALSplitter(425): Processed 26 edits across 1 Regions in 138 ms; skipped=1; WAL=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274-splitting/2998b885bdef%2C36075%2C1731482006274.meta.1731482021283.meta, size=14.0 K, length=14382, corrupted=false, cancelled=false 2024-11-13T07:14:15,271 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=38}] wal.WALSplitter(428): Completed split of hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274-splitting/2998b885bdef%2C36075%2C1731482006274.meta.1731482021283.meta, journal: Splitting hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274-splitting/2998b885bdef%2C36075%2C1731482006274.meta.1731482021283.meta, size=14.0 K (14382bytes) at 1731482055045Finishing writing output for hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274-splitting/2998b885bdef%2C36075%2C1731482006274.meta.1731482021283.meta so closing down at 1731482055148 (+103 ms)Creating recovered edits writer path=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2998b885bdef%2C36075%2C1731482006274.meta.1731482021283.meta.temp at 1731482055161 (+13 ms)3 split writer threads finished at 1731482055169 (+8 ms)Closed recovered edits writer path=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2998b885bdef%2C36075%2C1731482006274.meta.1731482021283.meta.temp (wrote 25 edits, skipped 0 edits in 2 ms) at 1731482055264 (+95 ms)Rename recovered edits hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2998b885bdef%2C36075%2C1731482006274.meta.1731482021283.meta.temp to hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/recovered.edits/0000000000000000028 at 1731482055267 (+3 ms)Processed 26 edits across 1 Regions in 138 ms; skipped=1; WAL=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274-splitting/2998b885bdef%2C36075%2C1731482006274.meta.1731482021283.meta, size=14.0 K, length=14382, corrupted=false, cancelled=false at 1731482055271 (+4 ms) 2024-11-13T07:14:15,271 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=38}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274-splitting/2998b885bdef%2C36075%2C1731482006274.meta.1731482021283.meta 2024-11-13T07:14:15,271 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=38}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=38 2024-11-13T07:14:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.HMaster(4169): Remote procedure done, pid=38 2024-11-13T07:14:15,280 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274-splitting/2998b885bdef%2C36075%2C1731482006274.meta.1731482021283.meta to hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/oldWALs 2024-11-13T07:14:15,284 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-13T07:14:15,284 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2998b885bdef%2C36075%2C1731482006274.meta.1731482021283.meta, worker=2998b885bdef,44417,1731482005920 in 9.9310 sec 2024-11-13T07:14:15,286 DEBUG [PEWorker-2 {}] master.SplitWALManager(172): Release split WAL worker=2998b885bdef,44417,1731482005920 2024-11-13T07:14:15,291 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-13T07:14:15,291 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; SplitWALProcedure 2998b885bdef%2C36075%2C1731482006274.meta.1731482021283.meta, worker=2998b885bdef,44417,1731482005920 in 9.9520 sec 2024-11-13T07:14:15,294 INFO [PEWorker-5 {}] master.SplitLogManager(171): hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274-splitting dir is empty, no logs to split. 2024-11-13T07:14:15,294 INFO [PEWorker-5 {}] master.SplitWALManager(105): 2998b885bdef,36075,1731482006274 WAL count=0, meta=true 2024-11-13T07:14:15,294 DEBUG [PEWorker-5 {}] procedure.ServerCrashProcedure(329): Check if 2998b885bdef,36075,1731482006274 WAL splitting is done? wals=0, meta=true 2024-11-13T07:14:15,297 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T07:14:15,298 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T07:14:15,299 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T07:14:15,305 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/afca0ae186274ac99f7499d163ba2c4e/.tmp/cf/b42be0ac08494614a648964fe6d9b775 as hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/afca0ae186274ac99f7499d163ba2c4e/cf/b42be0ac08494614a648964fe6d9b775 2024-11-13T07:14:15,305 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/8ee3029b3e33cc034b84b2d370d7f61d/.tmp/cf/51fade2042e24b6ba4e2ca43c74b884f as hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/8ee3029b3e33cc034b84b2d370d7f61d/cf/51fade2042e24b6ba4e2ca43c74b884f 2024-11-13T07:14:15,316 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/8ee3029b3e33cc034b84b2d370d7f61d/cf/51fade2042e24b6ba4e2ca43c74b884f, entries=1297, sequenceid=5, filesize=46.1 K 2024-11-13T07:14:15,316 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/afca0ae186274ac99f7499d163ba2c4e/cf/b42be0ac08494614a648964fe6d9b775, entries=731, sequenceid=5, filesize=28.0 K 2024-11-13T07:14:15,324 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3140): Finished flush of dataSize ~21.42 KB/21930, heapSize ~74.48 KB/76264, currentSize=0 B/0 for afca0ae186274ac99f7499d163ba2c4e in 313ms, sequenceid=5, compaction requested=false 2024-11-13T07:14:15,324 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=34}] regionserver.HRegion(3140): Finished flush of dataSize ~38.00 KB/38910, heapSize ~131.96 KB/135128, currentSize=0 B/0 for 8ee3029b3e33cc034b84b2d370d7f61d in 310ms, sequenceid=5, compaction requested=false 2024-11-13T07:14:15,324 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2603): Flush status journal for afca0ae186274ac99f7499d163ba2c4e: 2024-11-13T07:14:15,324 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=34}] regionserver.HRegion(2603): Flush status journal for 8ee3029b3e33cc034b84b2d370d7f61d: 2024-11-13T07:14:15,324 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e. for SnapshotProcedureTest completed. 2024-11-13T07:14:15,324 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=34}] regionserver.SnapshotRegionCallable(75): Snapshotting region SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d. for SnapshotProcedureTest completed. 2024-11-13T07:14:15,325 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=34}] snapshot.SnapshotManifest(241): Storing 'SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d.' region-info for snapshot=SnapshotProcedureTest 2024-11-13T07:14:15,325 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(241): Storing 'SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e.' region-info for snapshot=SnapshotProcedureTest 2024-11-13T07:14:15,325 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-13T07:14:15,325 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=34}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-13T07:14:15,325 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=34}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/8ee3029b3e33cc034b84b2d370d7f61d/cf/51fade2042e24b6ba4e2ca43c74b884f] hfiles 2024-11-13T07:14:15,325 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/afca0ae186274ac99f7499d163ba2c4e/cf/b42be0ac08494614a648964fe6d9b775] hfiles 2024-11-13T07:14:15,327 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/afca0ae186274ac99f7499d163ba2c4e/cf/b42be0ac08494614a648964fe6d9b775 for snapshot=SnapshotProcedureTest 2024-11-13T07:14:15,327 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=34}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/8ee3029b3e33cc034b84b2d370d7f61d/cf/51fade2042e24b6ba4e2ca43c74b884f for snapshot=SnapshotProcedureTest 2024-11-13T07:14:15,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741857_1033 (size=105) 2024-11-13T07:14:15,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741858_1034 (size=105) 2024-11-13T07:14:15,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741857_1033 (size=105) 2024-11-13T07:14:15,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741858_1034 (size=105) 2024-11-13T07:14:15,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741857_1033 (size=105) 2024-11-13T07:14:15,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741858_1034 (size=105) 2024-11-13T07:14:15,355 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=34}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d. 2024-11-13T07:14:15,355 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=34}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=34 2024-11-13T07:14:15,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.HMaster(4169): Remote procedure done, pid=34 2024-11-13T07:14:15,356 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] procedure.SnapshotRegionProcedure(132): finish snapshot SnapshotProcedureTest on region 8ee3029b3e33cc034b84b2d370d7f61d 2024-11-13T07:14:15,356 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 8ee3029b3e33cc034b84b2d370d7f61d 2024-11-13T07:14:15,360 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=25, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 8ee3029b3e33cc034b84b2d370d7f61d in 10.3900 sec 2024-11-13T07:14:15,360 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e. 2024-11-13T07:14:15,360 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-11-13T07:14:15,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-11-13T07:14:15,361 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] procedure.SnapshotRegionProcedure(132): finish snapshot SnapshotProcedureTest on region afca0ae186274ac99f7499d163ba2c4e 2024-11-13T07:14:15,361 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure afca0ae186274ac99f7499d163ba2c4e 2024-11-13T07:14:15,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=25, state=SUCCESS, hasLock=false; SnapshotRegionProcedure afca0ae186274ac99f7499d163ba2c4e in 10.3940 sec 2024-11-13T07:14:15,450 DEBUG [2998b885bdef:46239 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-13T07:14:15,450 DEBUG [2998b885bdef:46239 {}] balancer.BalancerClusterState(204): Hosts are {2998b885bdef=0} racks are {/default-rack=0} 2024-11-13T07:14:15,450 DEBUG [2998b885bdef:46239 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T07:14:15,450 DEBUG [2998b885bdef:46239 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T07:14:15,450 DEBUG [2998b885bdef:46239 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T07:14:15,450 DEBUG [2998b885bdef:46239 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T07:14:15,450 INFO [2998b885bdef:46239 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T07:14:15,451 INFO [2998b885bdef:46239 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T07:14:15,451 DEBUG [2998b885bdef:46239 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T07:14:15,451 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2998b885bdef,37717,1731482006937 2024-11-13T07:14:15,452 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2998b885bdef,37717,1731482006937, state=OPENING 2024-11-13T07:14:15,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101330aa3af0001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T07:14:15,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T07:14:15,494 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37717-0x101330aa3af0003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T07:14:15,494 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=39, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T07:14:15,494 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:14:15,494 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:14:15,495 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2998b885bdef,37717,1731482006937}] 2024-11-13T07:14:15,495 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:14:20,344 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:14:20,352 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 88b9c24c43eef7986e237d795658427e 2024-11-13T07:14:20,353 WARN [PEWorker-3 {}] procedure.SnapshotRegionProcedure(211): pid=35, ppid=25, state=RUNNABLE, hasLock=true; SnapshotRegionProcedure feb86c9c548b3a71cf3eef83fbb2adeb can not run currently because target server of region SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb. 2998b885bdef,36075,1731482006274 is in state SPLITTING_META_DONE, wait 16000 ms to retry 2024-11-13T07:14:20,353 INFO [PEWorker-3 {}] procedure2.TimeoutExecutorThread(82): ADDED pid=35, ppid=25, state=WAITING_TIMEOUT, hasLock=true; SnapshotRegionProcedure feb86c9c548b3a71cf3eef83fbb2adeb; timeout=16000, timestamp=1731482076353 2024-11-13T07:14:20,355 WARN [PEWorker-4 {}] procedure.SnapshotRegionProcedure(211): pid=29, ppid=25, state=RUNNABLE, hasLock=true; SnapshotRegionProcedure 88b9c24c43eef7986e237d795658427e can not run currently because target server of region SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e. 2998b885bdef,36075,1731482006274 is in state SPLITTING_META_DONE, wait 16000 ms to retry 2024-11-13T07:14:20,355 INFO [PEWorker-4 {}] procedure2.TimeoutExecutorThread(82): ADDED pid=29, ppid=25, state=WAITING_TIMEOUT, hasLock=true; SnapshotRegionProcedure 88b9c24c43eef7986e237d795658427e; timeout=16000, timestamp=1731482076355 2024-11-13T07:14:20,361 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f70ed29b296742bfa400816128951d24 2024-11-13T07:14:20,362 WARN [PEWorker-1 {}] procedure.SnapshotRegionProcedure(211): pid=32, ppid=25, state=RUNNABLE, hasLock=true; SnapshotRegionProcedure f70ed29b296742bfa400816128951d24 can not run currently because target server of region SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24. 2998b885bdef,36075,1731482006274 is in state SPLITTING_META_DONE, wait 16000 ms to retry 2024-11-13T07:14:20,362 INFO [PEWorker-1 {}] procedure2.TimeoutExecutorThread(82): ADDED pid=32, ppid=25, state=WAITING_TIMEOUT, hasLock=true; SnapshotRegionProcedure f70ed29b296742bfa400816128951d24; timeout=16000, timestamp=1731482076362 2024-11-13T07:14:21,686 INFO [master/2998b885bdef:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-13T07:14:21,686 INFO [master/2998b885bdef:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-13T07:14:23,296 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T07:14:25,511 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T07:14:25,511 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:14:25,511 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T07:14:25,514 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2998b885bdef%2C37717%2C1731482006937.meta, suffix=.meta, logDir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,37717,1731482006937, archiveDir=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/oldWALs, maxLogs=32 2024-11-13T07:14:25,530 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,37717,1731482006937/2998b885bdef%2C37717%2C1731482006937.meta.1731482065515.meta, exclude list is [], retry=0 2024-11-13T07:14:25,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38559,DS-f0b0f387-c446-4c2b-8ab1-3fda3b8c8206,DISK] 2024-11-13T07:14:25,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40257,DS-14e25fb3-d25b-4488-aa49-d869ee0f7b89,DISK] 2024-11-13T07:14:25,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38027,DS-77acad21-8279-4d7b-afc4-61162fa08ba0,DISK] 2024-11-13T07:14:25,538 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,37717,1731482006937/2998b885bdef%2C37717%2C1731482006937.meta.1731482065515.meta 2024-11-13T07:14:25,538 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43833:43833),(127.0.0.1/127.0.0.1:35195:35195),(127.0.0.1/127.0.0.1:40023:40023)] 2024-11-13T07:14:25,538 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:14:25,539 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T07:14:25,539 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T07:14:25,539 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T07:14:25,539 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T07:14:25,539 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:14:25,539 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T07:14:25,539 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T07:14:25,541 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T07:14:25,555 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T07:14:25,556 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:14:25,556 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:14:25,557 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T07:14:25,558 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T07:14:25,558 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:14:25,561 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:14:25,562 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T07:14:25,570 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T07:14:25,570 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:14:25,576 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:14:25,576 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T07:14:25,577 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T07:14:25,578 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:14:25,580 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:14:25,581 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T07:14:25,585 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740 2024-11-13T07:14:25,588 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740 2024-11-13T07:14:25,593 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/recovered.edits/0000000000000000028 2024-11-13T07:14:25,613 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.HRegion(5793): Applied 104, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=28, path=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/recovered.edits/0000000000000000028 2024-11-13T07:14:25,614 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=12.43 KB heapSize=21.09 KB 2024-11-13T07:14:25,639 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/.tmp/info/d6c7f49e0b6148eea3af6a89778961b2 is 167, key is SPTestTable,19999999,1731482021829.5f4b94b50abb029a06bde24412d357dd./info:regioninfo/1731482032923/Put/seqid=0 2024-11-13T07:14:25,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741860_1036 (size=18393) 2024-11-13T07:14:25,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741860_1036 (size=18393) 2024-11-13T07:14:25,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741860_1036 (size=18393) 2024-11-13T07:14:25,676 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.28 KB at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/.tmp/info/d6c7f49e0b6148eea3af6a89778961b2 2024-11-13T07:14:25,724 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/.tmp/ns/b0ab97db687c4ca080bfc994801129e8 is 43, key is default/ns:d/1731482021575/Put/seqid=0 2024-11-13T07:14:25,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741861_1037 (size=5153) 2024-11-13T07:14:25,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741861_1037 (size=5153) 2024-11-13T07:14:25,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741861_1037 (size=5153) 2024-11-13T07:14:25,781 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/.tmp/ns/b0ab97db687c4ca080bfc994801129e8 2024-11-13T07:14:25,839 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/.tmp/table/98c9d97100ed41c790b3df1eccd7b1ba is 47, key is SPTestTable/table:state/1731482033009/Put/seqid=0 2024-11-13T07:14:25,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741862_1038 (size=5214) 2024-11-13T07:14:25,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741862_1038 (size=5214) 2024-11-13T07:14:25,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741862_1038 (size=5214) 2024-11-13T07:14:25,926 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=86 B at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/.tmp/table/98c9d97100ed41c790b3df1eccd7b1ba 2024-11-13T07:14:25,977 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/.tmp/info/d6c7f49e0b6148eea3af6a89778961b2 as hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/info/d6c7f49e0b6148eea3af6a89778961b2 2024-11-13T07:14:26,014 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/info/d6c7f49e0b6148eea3af6a89778961b2, entries=100, sequenceid=28, filesize=18.0 K 2024-11-13T07:14:26,016 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/.tmp/ns/b0ab97db687c4ca080bfc994801129e8 as hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/ns/b0ab97db687c4ca080bfc994801129e8 2024-11-13T07:14:26,033 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/ns/b0ab97db687c4ca080bfc994801129e8, entries=2, sequenceid=28, filesize=5.0 K 2024-11-13T07:14:26,039 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/.tmp/table/98c9d97100ed41c790b3df1eccd7b1ba as hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/table/98c9d97100ed41c790b3df1eccd7b1ba 2024-11-13T07:14:26,075 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/table/98c9d97100ed41c790b3df1eccd7b1ba, entries=2, sequenceid=28, filesize=5.1 K 2024-11-13T07:14:26,075 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.HRegion(3140): Finished flush of dataSize ~12.43 KB/12732, heapSize ~20.80 KB/21296, currentSize=0 B/0 for 1588230740 in 461ms, sequenceid=28, compaction requested=false; wal=null 2024-11-13T07:14:26,080 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/recovered.edits/0000000000000000028 2024-11-13T07:14:26,094 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T07:14:26,094 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T07:14:26,100 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T07:14:26,107 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T07:14:26,114 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/recovered.edits/28.seqid, newMaxSeqId=28, maxSeqId=1 2024-11-13T07:14:26,115 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=29; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60837028, jitterRate=-0.09345763921737671}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T07:14:26,116 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T07:14:26,116 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731482065539Writing region info on filesystem at 1731482065539Initializing all the Stores at 1731482065541 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731482065541Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731482065541Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482065541Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731482065541Obtaining lock to block concurrent updates at 1731482065614 (+73 ms)Preparing flush snapshotting stores in 1588230740 at 1731482065614Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=12732, getHeapSize=21536, getOffHeapSize=0, getCellsCount=104 at 1731482065614Flushing stores of hbase:meta,,1.1588230740 at 1731482065614Flushing 1588230740/info: creating writer at 1731482065615 (+1 ms)Flushing 1588230740/info: appending metadata at 1731482065638 (+23 ms)Flushing 1588230740/info: closing flushed file at 1731482065638Flushing 1588230740/ns: creating writer at 1731482065697 (+59 ms)Flushing 1588230740/ns: appending metadata at 1731482065723 (+26 ms)Flushing 1588230740/ns: closing flushed file at 1731482065723Flushing 1588230740/table: creating writer at 1731482065796 (+73 ms)Flushing 1588230740/table: appending metadata at 1731482065832 (+36 ms)Flushing 1588230740/table: closing flushed file at 1731482065832Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69a1b442: reopening flushed file at 1731482065976 (+144 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53a7bc52: reopening flushed file at 1731482066014 (+38 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c72f287: reopening flushed file at 1731482066033 (+19 ms)Finished flush of dataSize ~12.43 KB/12732, heapSize ~20.80 KB/21296, currentSize=0 B/0 for 1588230740 in 461ms, sequenceid=28, compaction requested=false; wal=null at 1731482066075 (+42 ms)Cleaning up temporary data from old regions at 1731482066094 (+19 ms)Running coprocessor post-open hooks at 1731482066116 (+22 ms)Region opened successfully at 1731482066116 2024-11-13T07:14:26,117 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=40, masterSystemTime=1731482065497 2024-11-13T07:14:26,133 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T07:14:26,133 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=40}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T07:14:26,136 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=29, regionLocation=2998b885bdef,37717,1731482006937 2024-11-13T07:14:26,138 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2998b885bdef,37717,1731482006937, state=OPEN 2024-11-13T07:14:26,158 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46239 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=2998b885bdef,37717,1731482006937, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T07:14:26,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101330aa3af0001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T07:14:26,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T07:14:26,195 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37717-0x101330aa3af0003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T07:14:26,195 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:14:26,196 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:14:26,196 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=40, ppid=39, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2998b885bdef,37717,1731482006937 2024-11-13T07:14:26,198 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:14:26,205 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-13T07:14:26,205 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2998b885bdef,37717,1731482006937 in 10.7010 sec 2024-11-13T07:14:26,215 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=36 2024-11-13T07:14:26,215 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 10.9090 sec 2024-11-13T07:14:26,216 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(207): 2998b885bdef,36075,1731482006274 had 4 regions 2024-11-13T07:14:26,220 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=36, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 2998b885bdef,36075,1731482006274, splitWal=true, meta=true, isMeta: false 2024-11-13T07:14:26,237 INFO [PEWorker-5 {}] master.SplitWALManager(105): 2998b885bdef,36075,1731482006274 WAL count=1, meta=false 2024-11-13T07:14:26,237 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=36, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 2998b885bdef%2C36075%2C1731482006274.1731482010632}] 2024-11-13T07:14:26,241 DEBUG [PEWorker-4 {}] master.SplitWALManager(158): Acquired split WAL worker=2998b885bdef,37717,1731482006937 2024-11-13T07:14:26,243 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 2998b885bdef%2C36075%2C1731482006274.1731482010632, worker=2998b885bdef,37717,1731482006937}] 2024-11-13T07:14:30,549 INFO [regionserver/2998b885bdef:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: ExecutorStatusChore was stopped 2024-11-13T07:14:30,666 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=2998b885bdef:46239 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 10 more 2024-11-13T07:14:31,079 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T07:14:36,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=42 2024-11-13T07:14:36,278 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=42}] wal.WALSplitter(299): Splitting hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274-splitting/2998b885bdef%2C36075%2C1731482006274.1731482010632, size=966 (966bytes) 2024-11-13T07:14:36,279 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=42}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274-splitting/2998b885bdef%2C36075%2C1731482006274.1731482010632 2024-11-13T07:14:36,279 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=42}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274-splitting/2998b885bdef%2C36075%2C1731482006274.1731482010632 after 0ms 2024-11-13T07:14:36,283 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=42}] wal.WALSplitter(310): Open hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274-splitting/2998b885bdef%2C36075%2C1731482006274.1731482010632 took 5ms 2024-11-13T07:14:36,288 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=42}] wal.WALSplitter(352): Last flushed sequenceid for f70ed29b296742bfa400816128951d24: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "cf" sequence_id: 18446744073709551615 } 2024-11-13T07:14:36,291 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=42}] wal.WALSplitter(352): Last flushed sequenceid for 88b9c24c43eef7986e237d795658427e: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "cf" sequence_id: 18446744073709551615 } 2024-11-13T07:14:36,292 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=42}] wal.WALSplitter(352): Last flushed sequenceid for feb86c9c548b3a71cf3eef83fbb2adeb: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "cf" sequence_id: 18446744073709551615 } 2024-11-13T07:14:36,293 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=42}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274-splitting/2998b885bdef%2C36075%2C1731482006274.1731482010632 so closing down 2024-11-13T07:14:36,293 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=42}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T07:14:36,294 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=42}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T07:14:36,294 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=42}] wal.WALSplitter(425): Processed 3 edits across 0 Regions in 9 ms; skipped=3; WAL=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274-splitting/2998b885bdef%2C36075%2C1731482006274.1731482010632, size=966, length=966, corrupted=false, cancelled=false 2024-11-13T07:14:36,296 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=42}] wal.WALSplitter(428): Completed split of hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274-splitting/2998b885bdef%2C36075%2C1731482006274.1731482010632, journal: Splitting hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274-splitting/2998b885bdef%2C36075%2C1731482006274.1731482010632, size=966 (966bytes) at 1731482076278Finishing writing output for hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274-splitting/2998b885bdef%2C36075%2C1731482006274.1731482010632 so closing down at 1731482076293 (+15 ms)3 split writer threads finished at 1731482076294 (+1 ms)Processed 3 edits across 0 Regions in 9 ms; skipped=3; WAL=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274-splitting/2998b885bdef%2C36075%2C1731482006274.1731482010632, size=966, length=966, corrupted=false, cancelled=false at 1731482076296 (+2 ms) 2024-11-13T07:14:36,296 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=42}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274-splitting/2998b885bdef%2C36075%2C1731482006274.1731482010632 2024-11-13T07:14:36,296 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=42}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=42 2024-11-13T07:14:36,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.HMaster(4169): Remote procedure done, pid=42 2024-11-13T07:14:36,322 INFO [PEWorker-2 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274-splitting/2998b885bdef%2C36075%2C1731482006274.1731482010632 to hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/oldWALs 2024-11-13T07:14:36,326 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=41 2024-11-13T07:14:36,326 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=41, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2998b885bdef%2C36075%2C1731482006274.1731482010632, worker=2998b885bdef,37717,1731482006937 in 10.0790 sec 2024-11-13T07:14:36,341 DEBUG [PEWorker-2 {}] master.SplitWALManager(172): Release split WAL worker=2998b885bdef,37717,1731482006937 2024-11-13T07:14:36,346 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=36 2024-11-13T07:14:36,346 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=36, state=SUCCESS, hasLock=false; SplitWALProcedure 2998b885bdef%2C36075%2C1731482006274.1731482010632, worker=2998b885bdef,37717,1731482006937 in 10.1040 sec 2024-11-13T07:14:36,348 INFO [PEWorker-5 {}] master.SplitLogManager(171): hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/WALs/2998b885bdef,36075,1731482006274-splitting dir is empty, no logs to split. 2024-11-13T07:14:36,349 INFO [PEWorker-5 {}] master.SplitWALManager(105): 2998b885bdef,36075,1731482006274 WAL count=0, meta=false 2024-11-13T07:14:36,349 DEBUG [PEWorker-5 {}] procedure.ServerCrashProcedure(329): Check if 2998b885bdef,36075,1731482006274 WAL splitting is done? wals=0, meta=false 2024-11-13T07:14:36,353 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:14:36,353 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=88b9c24c43eef7986e237d795658427e, ASSIGN}, {pid=44, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=feb86c9c548b3a71cf3eef83fbb2adeb, ASSIGN}, {pid=45, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=f70ed29b296742bfa400816128951d24, ASSIGN}] 2024-11-13T07:14:36,355 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 88b9c24c43eef7986e237d795658427e 2024-11-13T07:14:36,356 WARN [PEWorker-4 {}] procedure.SnapshotRegionProcedure(211): pid=35, ppid=25, state=RUNNABLE, hasLock=true; SnapshotRegionProcedure feb86c9c548b3a71cf3eef83fbb2adeb can not run currently because region SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb. has a TRSP attached pid=44, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=feb86c9c548b3a71cf3eef83fbb2adeb, ASSIGN, wait 32000 ms to retry 2024-11-13T07:14:36,356 INFO [PEWorker-4 {}] procedure2.TimeoutExecutorThread(82): ADDED pid=35, ppid=25, state=WAITING_TIMEOUT, hasLock=true; SnapshotRegionProcedure feb86c9c548b3a71cf3eef83fbb2adeb; timeout=32000, timestamp=1731482108356 2024-11-13T07:14:36,358 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=45, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=f70ed29b296742bfa400816128951d24, ASSIGN 2024-11-13T07:14:36,359 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=44, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=feb86c9c548b3a71cf3eef83fbb2adeb, ASSIGN 2024-11-13T07:14:36,359 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(842): Waiting on xlock for pid=43, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=88b9c24c43eef7986e237d795658427e, ASSIGN held by pid=29 2024-11-13T07:14:36,359 DEBUG [PEWorker-2 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=43, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=88b9c24c43eef7986e237d795658427e, ASSIGN 2024-11-13T07:14:36,359 WARN [PEWorker-1 {}] procedure.SnapshotRegionProcedure(211): pid=29, ppid=25, state=RUNNABLE, hasLock=true; SnapshotRegionProcedure 88b9c24c43eef7986e237d795658427e can not run currently because region SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e. has a TRSP attached pid=43, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=88b9c24c43eef7986e237d795658427e, ASSIGN, wait 32000 ms to retry 2024-11-13T07:14:36,359 INFO [PEWorker-1 {}] procedure2.TimeoutExecutorThread(82): ADDED pid=29, ppid=25, state=WAITING_TIMEOUT, hasLock=true; SnapshotRegionProcedure 88b9c24c43eef7986e237d795658427e; timeout=32000, timestamp=1731482108359 2024-11-13T07:14:36,362 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=45, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=f70ed29b296742bfa400816128951d24, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T07:14:36,362 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(842): Waiting on xlock for pid=32, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f70ed29b296742bfa400816128951d24 held by pid=45 2024-11-13T07:14:36,362 DEBUG [PEWorker-2 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=32, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f70ed29b296742bfa400816128951d24 2024-11-13T07:14:36,362 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=44, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=feb86c9c548b3a71cf3eef83fbb2adeb, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T07:14:36,363 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=43, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=88b9c24c43eef7986e237d795658427e, ASSIGN 2024-11-13T07:14:36,365 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=43, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=88b9c24c43eef7986e237d795658427e, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T07:14:36,516 DEBUG [2998b885bdef:46239 {}] balancer.BalancerClusterState(204): Hosts are {2998b885bdef=0} racks are {/default-rack=0} 2024-11-13T07:14:36,517 DEBUG [2998b885bdef:46239 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T07:14:36,517 DEBUG [2998b885bdef:46239 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T07:14:36,517 DEBUG [2998b885bdef:46239 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T07:14:36,517 DEBUG [2998b885bdef:46239 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T07:14:36,517 INFO [2998b885bdef:46239 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T07:14:36,517 INFO [2998b885bdef:46239 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T07:14:36,517 DEBUG [2998b885bdef:46239 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T07:14:36,517 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=feb86c9c548b3a71cf3eef83fbb2adeb, regionState=OPENING, regionLocation=2998b885bdef,44417,1731482005920 2024-11-13T07:14:36,518 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=43 updating hbase:meta row=88b9c24c43eef7986e237d795658427e, regionState=OPENING, regionLocation=2998b885bdef,44417,1731482005920 2024-11-13T07:14:36,519 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=45 updating hbase:meta row=f70ed29b296742bfa400816128951d24, regionState=OPENING, regionLocation=2998b885bdef,37717,1731482006937 2024-11-13T07:14:36,519 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 2998b885bdef:36075 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2998b885bdef/172.17.0.3:36075 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:14:36,521 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2998b885bdef,36075,1731482006274, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2998b885bdef,36075,1731482006274, seqNum=-1, error=java.net.ConnectException: Call to address=2998b885bdef:36075 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2998b885bdef/172.17.0.3:36075 2024-11-13T07:14:36,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=2998b885bdef,36075,1731482006274, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T07:14:36,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=2998b885bdef,36075,1731482006274, seqNum=-1 from cache 2024-11-13T07:14:36,523 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 2998b885bdef:36075 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2998b885bdef/172.17.0.3:36075 2024-11-13T07:14:36,524 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 2998b885bdef:36075 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2998b885bdef/172.17.0.3:36075 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:14:36,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2998b885bdef,36075,1731482006274, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=2998b885bdef:36075 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2998b885bdef/172.17.0.3:36075 2024-11-13T07:14:36,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2998b885bdef,36075,1731482006274, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=2998b885bdef:36075 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2998b885bdef/172.17.0.3:36075 2024-11-13T07:14:36,526 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 2998b885bdef:36075 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2998b885bdef/172.17.0.3:36075 2024-11-13T07:14:36,632 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T07:14:36,632 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2998b885bdef,37717,1731482006937, seqNum=-1] 2024-11-13T07:14:36,632 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T07:14:36,635 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40721, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T07:14:36,642 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=feb86c9c548b3a71cf3eef83fbb2adeb, ASSIGN because future has completed 2024-11-13T07:14:36,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=44, state=RUNNABLE, hasLock=false; OpenRegionProcedure feb86c9c548b3a71cf3eef83fbb2adeb, server=2998b885bdef,44417,1731482005920}] 2024-11-13T07:14:36,644 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=88b9c24c43eef7986e237d795658427e, ASSIGN because future has completed 2024-11-13T07:14:36,644 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=43, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88b9c24c43eef7986e237d795658427e, server=2998b885bdef,44417,1731482005920}] 2024-11-13T07:14:36,658 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=f70ed29b296742bfa400816128951d24, ASSIGN because future has completed 2024-11-13T07:14:36,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=45, state=RUNNABLE, hasLock=false; OpenRegionProcedure f70ed29b296742bfa400816128951d24, server=2998b885bdef,37717,1731482006937}] 2024-11-13T07:14:37,559 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 8ee3029b3e33cc034b84b2d370d7f61d, had cached 0 bytes from a total of 47197 2024-11-13T07:14:37,794 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region afca0ae186274ac99f7499d163ba2c4e, had cached 0 bytes from a total of 28623 2024-11-13T07:14:41,649 DEBUG [master/2998b885bdef:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5f4b94b50abb029a06bde24412d357dd changed from -1.0 to 0.0, refreshing cache 2024-11-13T07:14:41,650 DEBUG [master/2998b885bdef:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6af43b5a45829fff6c10e65235eba1d3 changed from -1.0 to 0.0, refreshing cache 2024-11-13T07:14:41,650 DEBUG [master/2998b885bdef:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8ee3029b3e33cc034b84b2d370d7f61d changed from -1.0 to 0.0, refreshing cache 2024-11-13T07:14:41,651 DEBUG [master/2998b885bdef:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-13T07:14:41,651 DEBUG [master/2998b885bdef:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c5c88a6d20592c1b4adcc2269ffce351 changed from -1.0 to 0.0, refreshing cache 2024-11-13T07:14:41,651 DEBUG [master/2998b885bdef:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4b6c6542ce1ea1c1d3d96b4831e7c74a changed from -1.0 to 0.0, refreshing cache 2024-11-13T07:14:41,651 DEBUG [master/2998b885bdef:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c880039bd4ef8e05f593aab556d82223 changed from -1.0 to 0.0, refreshing cache 2024-11-13T07:14:41,652 DEBUG [master/2998b885bdef:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region afca0ae186274ac99f7499d163ba2c4e changed from -1.0 to 0.0, refreshing cache 2024-11-13T07:14:46,667 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e. 2024-11-13T07:14:46,667 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 88b9c24c43eef7986e237d795658427e, NAME => 'SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e.', STARTKEY => '4ccccccb', ENDKEY => '66666664'} 2024-11-13T07:14:46,668 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table SPTestTable 88b9c24c43eef7986e237d795658427e 2024-11-13T07:14:46,668 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:14:46,668 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 88b9c24c43eef7986e237d795658427e 2024-11-13T07:14:46,668 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 88b9c24c43eef7986e237d795658427e 2024-11-13T07:14:46,671 INFO [StoreOpener-88b9c24c43eef7986e237d795658427e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region 88b9c24c43eef7986e237d795658427e 2024-11-13T07:14:46,674 INFO [StoreOpener-88b9c24c43eef7986e237d795658427e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88b9c24c43eef7986e237d795658427e columnFamilyName cf 2024-11-13T07:14:46,674 DEBUG [StoreOpener-88b9c24c43eef7986e237d795658427e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:14:46,675 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24. 2024-11-13T07:14:46,675 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => f70ed29b296742bfa400816128951d24, NAME => 'SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24.', STARTKEY => '99999996', ENDKEY => 'b333332f'} 2024-11-13T07:14:46,676 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table SPTestTable f70ed29b296742bfa400816128951d24 2024-11-13T07:14:46,676 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:14:46,676 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for f70ed29b296742bfa400816128951d24 2024-11-13T07:14:46,676 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for f70ed29b296742bfa400816128951d24 2024-11-13T07:14:46,676 INFO [StoreOpener-88b9c24c43eef7986e237d795658427e-1 {}] regionserver.HStore(327): Store=88b9c24c43eef7986e237d795658427e/cf, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:14:46,676 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 88b9c24c43eef7986e237d795658427e 2024-11-13T07:14:46,678 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/88b9c24c43eef7986e237d795658427e 2024-11-13T07:14:46,679 INFO [StoreOpener-f70ed29b296742bfa400816128951d24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region f70ed29b296742bfa400816128951d24 2024-11-13T07:14:46,680 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/88b9c24c43eef7986e237d795658427e 2024-11-13T07:14:46,681 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 88b9c24c43eef7986e237d795658427e 2024-11-13T07:14:46,681 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 88b9c24c43eef7986e237d795658427e 2024-11-13T07:14:46,682 INFO [StoreOpener-f70ed29b296742bfa400816128951d24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f70ed29b296742bfa400816128951d24 columnFamilyName cf 2024-11-13T07:14:46,682 DEBUG [StoreOpener-f70ed29b296742bfa400816128951d24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:14:46,684 INFO [StoreOpener-f70ed29b296742bfa400816128951d24-1 {}] regionserver.HStore(327): Store=f70ed29b296742bfa400816128951d24/cf, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:14:46,684 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for f70ed29b296742bfa400816128951d24 2024-11-13T07:14:46,685 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 88b9c24c43eef7986e237d795658427e 2024-11-13T07:14:46,685 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/f70ed29b296742bfa400816128951d24 2024-11-13T07:14:46,686 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 88b9c24c43eef7986e237d795658427e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62164933, jitterRate=-0.07367031276226044}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:14:46,686 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 88b9c24c43eef7986e237d795658427e 2024-11-13T07:14:46,686 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 88b9c24c43eef7986e237d795658427e: Running coprocessor pre-open hook at 1731482086669Writing region info on filesystem at 1731482086669Initializing all the Stores at 1731482086671 (+2 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482086671Cleaning up temporary data from old regions at 1731482086681 (+10 ms)Running coprocessor post-open hooks at 1731482086686 (+5 ms)Region opened successfully at 1731482086686 2024-11-13T07:14:46,687 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e., pid=47, masterSystemTime=1731482086660 2024-11-13T07:14:46,689 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/f70ed29b296742bfa400816128951d24 2024-11-13T07:14:46,689 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for f70ed29b296742bfa400816128951d24 2024-11-13T07:14:46,689 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for f70ed29b296742bfa400816128951d24 2024-11-13T07:14:46,690 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e. 2024-11-13T07:14:46,690 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e. 2024-11-13T07:14:46,691 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb. 2024-11-13T07:14:46,691 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => feb86c9c548b3a71cf3eef83fbb2adeb, NAME => 'SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb.', STARTKEY => 'e6666661', ENDKEY => ''} 2024-11-13T07:14:46,691 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=43 updating hbase:meta row=88b9c24c43eef7986e237d795658427e, regionState=OPEN, openSeqNum=2, regionLocation=2998b885bdef,44417,1731482005920 2024-11-13T07:14:46,691 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table SPTestTable feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:14:46,691 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:14:46,692 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:14:46,692 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:14:46,694 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for f70ed29b296742bfa400816128951d24 2024-11-13T07:14:46,695 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=43, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88b9c24c43eef7986e237d795658427e, server=2998b885bdef,44417,1731482005920 because future has completed 2024-11-13T07:14:46,698 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened f70ed29b296742bfa400816128951d24; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64785809, jitterRate=-0.03461621701717377}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:14:46,698 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f70ed29b296742bfa400816128951d24 2024-11-13T07:14:46,698 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for f70ed29b296742bfa400816128951d24: Running coprocessor pre-open hook at 1731482086676Writing region info on filesystem at 1731482086676Initializing all the Stores at 1731482086678 (+2 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482086678Cleaning up temporary data from old regions at 1731482086689 (+11 ms)Running coprocessor post-open hooks at 1731482086698 (+9 ms)Region opened successfully at 1731482086698 2024-11-13T07:14:46,699 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24., pid=48, masterSystemTime=1731482086666 2024-11-13T07:14:46,700 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=43 2024-11-13T07:14:46,700 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=43, state=SUCCESS, hasLock=false; OpenRegionProcedure 88b9c24c43eef7986e237d795658427e, server=2998b885bdef,44417,1731482005920 in 10.0530 sec 2024-11-13T07:14:46,701 INFO [StoreOpener-feb86c9c548b3a71cf3eef83fbb2adeb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:14:46,702 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24. 2024-11-13T07:14:46,702 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24. 2024-11-13T07:14:46,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=88b9c24c43eef7986e237d795658427e, ASSIGN in 10.3470 sec 2024-11-13T07:14:46,703 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=45 updating hbase:meta row=f70ed29b296742bfa400816128951d24, regionState=OPEN, openSeqNum=2, regionLocation=2998b885bdef,37717,1731482006937 2024-11-13T07:14:46,706 INFO [StoreOpener-feb86c9c548b3a71cf3eef83fbb2adeb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region feb86c9c548b3a71cf3eef83fbb2adeb columnFamilyName cf 2024-11-13T07:14:46,706 DEBUG [StoreOpener-feb86c9c548b3a71cf3eef83fbb2adeb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:14:46,707 INFO [StoreOpener-feb86c9c548b3a71cf3eef83fbb2adeb-1 {}] regionserver.HStore(327): Store=feb86c9c548b3a71cf3eef83fbb2adeb/cf, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:14:46,707 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:14:46,708 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=45, state=RUNNABLE, hasLock=false; OpenRegionProcedure f70ed29b296742bfa400816128951d24, server=2998b885bdef,37717,1731482006937 because future has completed 2024-11-13T07:14:46,709 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:14:46,710 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46239 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=2998b885bdef,37717,1731482006937, table=SPTestTable, region=f70ed29b296742bfa400816128951d24. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T07:14:46,711 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:14:46,713 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:14:46,713 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:14:46,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=45 2024-11-13T07:14:46,715 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=45, state=SUCCESS, hasLock=false; OpenRegionProcedure f70ed29b296742bfa400816128951d24, server=2998b885bdef,37717,1731482006937 in 10.0510 sec 2024-11-13T07:14:46,717 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:14:46,718 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=f70ed29b296742bfa400816128951d24, ASSIGN in 10.3610 sec 2024-11-13T07:14:46,718 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f70ed29b296742bfa400816128951d24 2024-11-13T07:14:46,719 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened feb86c9c548b3a71cf3eef83fbb2adeb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68208460, jitterRate=0.016385257244110107}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:14:46,719 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:14:46,720 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for feb86c9c548b3a71cf3eef83fbb2adeb: Running coprocessor pre-open hook at 1731482086692Writing region info on filesystem at 1731482086692Initializing all the Stores at 1731482086694 (+2 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482086695 (+1 ms)Cleaning up temporary data from old regions at 1731482086713 (+18 ms)Running coprocessor post-open hooks at 1731482086719 (+6 ms)Region opened successfully at 1731482086719 2024-11-13T07:14:46,722 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb., pid=46, masterSystemTime=1731482086660 2024-11-13T07:14:46,725 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb. 2024-11-13T07:14:46,725 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb. 2024-11-13T07:14:46,726 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=44 updating hbase:meta row=feb86c9c548b3a71cf3eef83fbb2adeb, regionState=OPEN, openSeqNum=2, regionLocation=2998b885bdef,44417,1731482005920 2024-11-13T07:14:46,728 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=44, state=RUNNABLE, hasLock=false; OpenRegionProcedure feb86c9c548b3a71cf3eef83fbb2adeb, server=2998b885bdef,44417,1731482005920 because future has completed 2024-11-13T07:14:46,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=44 2024-11-13T07:14:46,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=44, state=SUCCESS, hasLock=false; OpenRegionProcedure feb86c9c548b3a71cf3eef83fbb2adeb, server=2998b885bdef,44417,1731482005920 in 10.0890 sec 2024-11-13T07:14:46,739 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=36 2024-11-13T07:14:46,739 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 2998b885bdef,36075,1731482006274 after splitting done 2024-11-13T07:14:46,739 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=feb86c9c548b3a71cf3eef83fbb2adeb, ASSIGN in 10.3820 sec 2024-11-13T07:14:46,743 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; ServerCrashProcedure 2998b885bdef,36075,1731482006274, splitWal=true, meta=true in 41.4420 sec 2024-11-13T07:14:52,197 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T07:14:53,296 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T07:14:56,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=32 2024-11-13T07:14:56,722 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=32}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24. 2024-11-13T07:14:56,722 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=32}] regionserver.HRegion(2603): Flush status journal for f70ed29b296742bfa400816128951d24: 2024-11-13T07:14:56,722 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=32}] regionserver.SnapshotRegionCallable(75): Snapshotting region SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24. for SnapshotProcedureTest completed. 2024-11-13T07:14:56,722 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=32}] snapshot.SnapshotManifest(241): Storing 'SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24.' region-info for snapshot=SnapshotProcedureTest 2024-11-13T07:14:56,722 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=32}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-13T07:14:56,722 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=32}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-13T07:14:56,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741863_1039 (size=65) 2024-11-13T07:14:56,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741863_1039 (size=65) 2024-11-13T07:14:56,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741863_1039 (size=65) 2024-11-13T07:14:56,741 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=32}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24. 2024-11-13T07:14:56,742 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=32 2024-11-13T07:14:56,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.HMaster(4169): Remote procedure done, pid=32 2024-11-13T07:14:56,742 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] procedure.SnapshotRegionProcedure(132): finish snapshot SnapshotProcedureTest on region f70ed29b296742bfa400816128951d24 2024-11-13T07:14:56,742 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure f70ed29b296742bfa400816128951d24 2024-11-13T07:14:56,746 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=25, state=SUCCESS, hasLock=false; SnapshotRegionProcedure f70ed29b296742bfa400816128951d24 in 51.7760 sec 2024-11-13T07:15:02,668 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=2998b885bdef:46239 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 10 more 2024-11-13T07:15:08,356 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:15:08,359 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 88b9c24c43eef7986e237d795658427e 2024-11-13T07:15:10,539 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 28760 2024-11-13T07:15:16,721 ERROR [ProcedureDispatcherTimeoutThread {}] procedure2.RemoteProcedureDispatcher$TimeoutExecutorThread(331): DelayQueue for RemoteProcedureDispatcher is not empty when timed waiting elapsed. If this is repeated consistently, it means no element is getting expired from the queue and it might freeze the system. Queue: [containedObject=2998b885bdef,44417,1731482005920, timeout=1731482118357, delay=1636, operations=[pid=35, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure feb86c9c548b3a71cf3eef83fbb2adeb, pid=29, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 88b9c24c43eef7986e237d795658427e]] 2024-11-13T07:15:18,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2024-11-13T07:15:18,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=29 2024-11-13T07:15:18,358 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb. 2024-11-13T07:15:18,359 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=29}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e. 2024-11-13T07:15:18,359 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for feb86c9c548b3a71cf3eef83fbb2adeb: 2024-11-13T07:15:18,359 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb. for SnapshotProcedureTest completed. 2024-11-13T07:15:18,359 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=29}] regionserver.HRegion(2603): Flush status journal for 88b9c24c43eef7986e237d795658427e: 2024-11-13T07:15:18,359 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=29}] regionserver.SnapshotRegionCallable(75): Snapshotting region SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e. for SnapshotProcedureTest completed. 2024-11-13T07:15:18,359 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(241): Storing 'SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb.' region-info for snapshot=SnapshotProcedureTest 2024-11-13T07:15:18,359 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=29}] snapshot.SnapshotManifest(241): Storing 'SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e.' region-info for snapshot=SnapshotProcedureTest 2024-11-13T07:15:18,359 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-13T07:15:18,359 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=29}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-13T07:15:18,359 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-13T07:15:18,359 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=29}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-13T07:15:18,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741865_1041 (size=65) 2024-11-13T07:15:18,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741864_1040 (size=57) 2024-11-13T07:15:18,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741865_1041 (size=65) 2024-11-13T07:15:18,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741865_1041 (size=65) 2024-11-13T07:15:18,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741864_1040 (size=57) 2024-11-13T07:15:18,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741864_1040 (size=57) 2024-11-13T07:15:18,377 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb. 2024-11-13T07:15:18,377 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-13T07:15:18,377 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=29}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e. 2024-11-13T07:15:18,377 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=29 2024-11-13T07:15:18,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-13T07:15:18,379 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] procedure.SnapshotRegionProcedure(132): finish snapshot SnapshotProcedureTest on region feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:15:18,380 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:15:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.HMaster(4169): Remote procedure done, pid=29 2024-11-13T07:15:18,380 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] procedure.SnapshotRegionProcedure(132): finish snapshot SnapshotProcedureTest on region 88b9c24c43eef7986e237d795658427e 2024-11-13T07:15:18,380 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 88b9c24c43eef7986e237d795658427e 2024-11-13T07:15:18,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=25, state=SUCCESS, hasLock=false; SnapshotRegionProcedure feb86c9c548b3a71cf3eef83fbb2adeb in 1 mins, 13.413 sec 2024-11-13T07:15:18,387 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=25 2024-11-13T07:15:18,387 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=25, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=25, snapshot={ ss=SnapshotProcedureTest table=SPTestTable type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-13T07:15:18,387 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=25, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 88b9c24c43eef7986e237d795658427e in 1 mins, 13.413 sec 2024-11-13T07:15:18,389 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=25, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=25, snapshot={ ss=SnapshotProcedureTest table=SPTestTable type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-13T07:15:18,391 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=25, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=25, snapshot={ ss=SnapshotProcedureTest table=SPTestTable type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-13T07:15:18,391 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for SnapshotProcedureTest 2024-11-13T07:15:18,395 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/.hbase-snapshot/.tmp/SnapshotProcedureTest 2024-11-13T07:15:18,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741866_1042 (size=1094) 2024-11-13T07:15:18,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741866_1042 (size=1094) 2024-11-13T07:15:18,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741866_1042 (size=1094) 2024-11-13T07:15:18,420 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=25, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=25, snapshot={ ss=SnapshotProcedureTest table=SPTestTable type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-13T07:15:18,429 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=25, state=RUNNABLE, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest}, {pid=50, ppid=25, state=RUNNABLE, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest}, {pid=51, ppid=25, state=RUNNABLE, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest}, {pid=52, ppid=25, state=RUNNABLE, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest}, {pid=53, ppid=25, state=RUNNABLE, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest}, {pid=54, ppid=25, state=RUNNABLE, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest}, {pid=55, ppid=25, state=RUNNABLE, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest}, {pid=56, ppid=25, state=RUNNABLE, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest}, {pid=57, ppid=25, state=RUNNABLE, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest}, {pid=58, ppid=25, state=RUNNABLE, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest}] 2024-11-13T07:15:18,437 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1464): pid=56, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest Acquired verify snapshot worker=2998b885bdef,37717,1731482006937 2024-11-13T07:15:18,437 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1464): pid=54, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest Acquired verify snapshot worker=2998b885bdef,44417,1731482005920 2024-11-13T07:15:18,437 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1464): pid=57, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest Acquired verify snapshot worker=2998b885bdef,44417,1731482005920 2024-11-13T07:15:18,437 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1464): pid=58, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest Acquired verify snapshot worker=2998b885bdef,44417,1731482005920 2024-11-13T07:15:18,437 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1464): pid=55, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest Acquired verify snapshot worker=2998b885bdef,37717,1731482006937 2024-11-13T07:15:18,442 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1464): pid=53, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest Acquired verify snapshot worker=2998b885bdef,37717,1731482006937 2024-11-13T07:15:22,559 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 8ee3029b3e33cc034b84b2d370d7f61d, had cached 0 bytes from a total of 47197 2024-11-13T07:15:22,794 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region afca0ae186274ac99f7499d163ba2c4e, had cached 0 bytes from a total of 28623 2024-11-13T07:15:23,296 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T07:15:28,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotVerifyCallable, pid=56 2024-11-13T07:15:28,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotVerifyCallable, pid=53 2024-11-13T07:15:28,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotVerifyCallable, pid=55 2024-11-13T07:15:28,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotVerifyCallable, pid=57 2024-11-13T07:15:28,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotVerifyCallable, pid=54 2024-11-13T07:15:28,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotVerifyCallable, pid=58 2024-11-13T07:15:28,464 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_VERIFY_SNAPSHOT, pid=56}] regionserver.RSSnapshotVerifier$SnapshotManifestCacheLoader(144): loading snapshot manifest for SnapshotProcedureTest from hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/.hbase-snapshot/.tmp/SnapshotProcedureTest 2024-11-13T07:15:28,466 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_VERIFY_SNAPSHOT, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-13T07:15:28,466 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_VERIFY_SNAPSHOT, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-13T07:15:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-13T07:15:28,469 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_VERIFY_SNAPSHOT, pid=56}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=56 2024-11-13T07:15:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-13T07:15:28,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.HMaster(4169): Remote procedure done, pid=56 2024-11-13T07:15:28,478 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1473): pid=53, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,37717,1731482006937 Release verify snapshot worker=2998b885bdef,37717,1731482006937 2024-11-13T07:15:28,480 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1473): pid=55, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,37717,1731482006937 Release verify snapshot worker=2998b885bdef,37717,1731482006937 2024-11-13T07:15:28,481 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_VERIFY_SNAPSHOT, pid=57}] regionserver.RSSnapshotVerifier$SnapshotManifestCacheLoader(144): loading snapshot manifest for SnapshotProcedureTest from hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/.hbase-snapshot/.tmp/SnapshotProcedureTest 2024-11-13T07:15:28,482 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_VERIFY_SNAPSHOT, pid=58}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=58 2024-11-13T07:15:28,482 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_VERIFY_SNAPSHOT, pid=54}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=54 2024-11-13T07:15:28,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.HMaster(4169): Remote procedure done, pid=58 2024-11-13T07:15:28,483 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_VERIFY_SNAPSHOT, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-13T07:15:28,485 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=25, state=SUCCESS, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,37717,1731482006937 in 10.0510 sec 2024-11-13T07:15:28,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.HMaster(4169): Remote procedure done, pid=54 2024-11-13T07:15:28,486 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=25, state=SUCCESS, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,37717,1731482006937 in 10.0500 sec 2024-11-13T07:15:28,487 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1473): pid=56, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,37717,1731482006937 Release verify snapshot worker=2998b885bdef,37717,1731482006937 2024-11-13T07:15:28,489 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1473): pid=58, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,44417,1731482005920 Release verify snapshot worker=2998b885bdef,44417,1731482005920 2024-11-13T07:15:28,489 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1464): pid=50, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest Acquired verify snapshot worker=2998b885bdef,44417,1731482005920 2024-11-13T07:15:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-13T07:15:28,492 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1464): pid=51, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest Acquired verify snapshot worker=2998b885bdef,37717,1731482006937 2024-11-13T07:15:28,492 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1464): pid=52, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest Acquired verify snapshot worker=2998b885bdef,37717,1731482006937 2024-11-13T07:15:28,497 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=25, state=SUCCESS, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,37717,1731482006937 in 10.0580 sec 2024-11-13T07:15:28,500 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1473): pid=57, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,44417,1731482005920 Release verify snapshot worker=2998b885bdef,44417,1731482005920 2024-11-13T07:15:28,506 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1473): pid=54, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,44417,1731482005920 Release verify snapshot worker=2998b885bdef,44417,1731482005920 2024-11-13T07:15:28,512 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=25, state=SUCCESS, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,44417,1731482005920 in 10.0710 sec 2024-11-13T07:15:28,514 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1464): pid=49, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest Acquired verify snapshot worker=2998b885bdef,37717,1731482006937 2024-11-13T07:15:28,514 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=25, state=SUCCESS, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,44417,1731482005920 in 10.0600 sec 2024-11-13T07:15:28,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=25, state=SUCCESS, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,44417,1731482005920 in 10.0770 sec 2024-11-13T07:15:30,332 INFO [regionserver/2998b885bdef:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactedHFilesCleaner was stopped 2024-11-13T07:15:38,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44417 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotVerifyCallable, pid=50 2024-11-13T07:15:38,491 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_VERIFY_SNAPSHOT, pid=50}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=50 2024-11-13T07:15:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.HMaster(4169): Remote procedure done, pid=50 2024-11-13T07:15:38,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotVerifyCallable, pid=49 2024-11-13T07:15:38,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotVerifyCallable, pid=52 2024-11-13T07:15:38,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37717 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotVerifyCallable, pid=51 2024-11-13T07:15:38,494 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_VERIFY_SNAPSHOT, pid=52}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=52 2024-11-13T07:15:38,494 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_VERIFY_SNAPSHOT, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-13T07:15:38,494 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_VERIFY_SNAPSHOT, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-13T07:15:38,495 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1473): pid=50, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,44417,1731482005920 Release verify snapshot worker=2998b885bdef,44417,1731482005920 2024-11-13T07:15:38,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.HMaster(4169): Remote procedure done, pid=52 2024-11-13T07:15:38,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-13T07:15:38,498 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=25, state=SUCCESS, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,44417,1731482005920 in 20.0660 sec 2024-11-13T07:15:38,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46239 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-13T07:15:38,503 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1473): pid=52, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,37717,1731482006937 Release verify snapshot worker=2998b885bdef,37717,1731482006937 2024-11-13T07:15:38,503 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1473): pid=51, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,37717,1731482006937 Release verify snapshot worker=2998b885bdef,37717,1731482006937 2024-11-13T07:15:38,504 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1473): pid=49, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,37717,1731482006937 Release verify snapshot worker=2998b885bdef,37717,1731482006937 2024-11-13T07:15:38,506 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=25, state=SUCCESS, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,37717,1731482006937 in 20.0740 sec 2024-11-13T07:15:38,506 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=25, state=SUCCESS, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,37717,1731482006937 in 20.0740 sec 2024-11-13T07:15:38,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=25 2024-11-13T07:15:38,508 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=25, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=25, snapshot={ ss=SnapshotProcedureTest table=SPTestTable type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-13T07:15:38,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=25, state=SUCCESS, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,37717,1731482006937 in 20.0750 sec 2024-11-13T07:15:38,509 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/.hbase-snapshot/.tmp/SnapshotProcedureTest to hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/.hbase-snapshot/SnapshotProcedureTest 2024-11-13T07:15:38,511 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=25, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=25, snapshot={ ss=SnapshotProcedureTest table=SPTestTable type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-13T07:15:38,511 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=SnapshotProcedureTest table=SPTestTable type=FLUSH ttl=0 }, snapshot procedure id = 25 2024-11-13T07:15:38,513 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=25, snapshot={ ss=SnapshotProcedureTest table=SPTestTable type=FLUSH ttl=0 } in 1 mins, 33.729 sec 2024-11-13T07:15:38,723 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=SPTestTable,, stopping at row=SPTestTable ,, for max=2147483647 with caching=100 2024-11-13T07:15:38,725 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 2998b885bdef:36075 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 2998b885bdef/172.17.0.3:36075 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T07:15:38,726 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2998b885bdef,36075,1731482006274, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2998b885bdef,36075,1731482006274, seqNum=-1, error=java.net.ConnectException: Call to address=2998b885bdef:36075 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 2998b885bdef/172.17.0.3:36075 2024-11-13T07:15:38,726 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=2998b885bdef,36075,1731482006274, seqNum=-1 is java.net.ConnectException: Connection refused 2024-11-13T07:15:38,726 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=2998b885bdef,36075,1731482006274, seqNum=-1 from cache 2024-11-13T07:15:38,727 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address 2998b885bdef:36075 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 2998b885bdef/172.17.0.3:36075 2024-11-13T07:15:38,832 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T07:15:38,834 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2998b885bdef,37717,1731482006937, seqNum=-1] 2024-11-13T07:15:38,867 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-13T07:15:38,867 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T07:15:38,867 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T07:15:38,868 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.procedure.TestSnapshotProcedure.teardown(TestSnapshotProcedure.java:151) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T07:15:38,868 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:15:38,868 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:15:38,868 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T07:15:38,868 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T07:15:38,868 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1025471862, stopped=false 2024-11-13T07:15:38,869 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=2998b885bdef,46239,1731482004610 2024-11-13T07:15:38,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101330aa3af0001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T07:15:38,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T07:15:38,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:15:38,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101330aa3af0001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:15:38,927 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T07:15:38,927 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37717-0x101330aa3af0003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T07:15:38,928 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37717-0x101330aa3af0003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:15:38,928 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T07:15:38,929 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.procedure.TestSnapshotProcedure.teardown(TestSnapshotProcedure.java:151) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T07:15:38,929 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:15:38,930 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44417-0x101330aa3af0001, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T07:15:38,930 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T07:15:38,931 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37717-0x101330aa3af0003, quorum=127.0.0.1:56395, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T07:15:38,931 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2998b885bdef,44417,1731482005920' ***** 2024-11-13T07:15:38,931 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T07:15:38,931 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2998b885bdef,37717,1731482006937' ***** 2024-11-13T07:15:38,932 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T07:15:38,932 INFO [RS:0;2998b885bdef:44417 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T07:15:38,932 INFO [RS:2;2998b885bdef:37717 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T07:15:38,932 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T07:15:38,932 INFO [RS:2;2998b885bdef:37717 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T07:15:38,932 INFO [RS:0;2998b885bdef:44417 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T07:15:38,932 INFO [RS:2;2998b885bdef:37717 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T07:15:38,932 INFO [RS:0;2998b885bdef:44417 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T07:15:38,932 INFO [RS:2;2998b885bdef:37717 {}] regionserver.HRegionServer(3091): Received CLOSE for 5f4b94b50abb029a06bde24412d357dd 2024-11-13T07:15:38,932 INFO [RS:0;2998b885bdef:44417 {}] regionserver.HRegionServer(3091): Received CLOSE for c880039bd4ef8e05f593aab556d82223 2024-11-13T07:15:38,933 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T07:15:38,933 INFO [RS:2;2998b885bdef:37717 {}] regionserver.HRegionServer(3091): Received CLOSE for 8ee3029b3e33cc034b84b2d370d7f61d 2024-11-13T07:15:38,933 INFO [RS:2;2998b885bdef:37717 {}] regionserver.HRegionServer(3091): Received CLOSE for 6af43b5a45829fff6c10e65235eba1d3 2024-11-13T07:15:38,933 INFO [RS:0;2998b885bdef:44417 {}] regionserver.HRegionServer(3091): Received CLOSE for 88b9c24c43eef7986e237d795658427e 2024-11-13T07:15:38,933 INFO [RS:2;2998b885bdef:37717 {}] regionserver.HRegionServer(3091): Received CLOSE for f70ed29b296742bfa400816128951d24 2024-11-13T07:15:38,933 INFO [RS:0;2998b885bdef:44417 {}] regionserver.HRegionServer(3091): Received CLOSE for c5c88a6d20592c1b4adcc2269ffce351 2024-11-13T07:15:38,933 INFO [RS:0;2998b885bdef:44417 {}] regionserver.HRegionServer(3091): Received CLOSE for afca0ae186274ac99f7499d163ba2c4e 2024-11-13T07:15:38,933 INFO [RS:2;2998b885bdef:37717 {}] regionserver.HRegionServer(959): stopping server 2998b885bdef,37717,1731482006937 2024-11-13T07:15:38,933 INFO [RS:2;2998b885bdef:37717 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T07:15:38,933 INFO [RS:0;2998b885bdef:44417 {}] regionserver.HRegionServer(3091): Received CLOSE for 4b6c6542ce1ea1c1d3d96b4831e7c74a 2024-11-13T07:15:38,933 INFO [RS:2;2998b885bdef:37717 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;2998b885bdef:37717. 2024-11-13T07:15:38,933 INFO [RS:0;2998b885bdef:44417 {}] regionserver.HRegionServer(3091): Received CLOSE for feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:15:38,933 INFO [RS:0;2998b885bdef:44417 {}] regionserver.HRegionServer(959): stopping server 2998b885bdef,44417,1731482005920 2024-11-13T07:15:38,933 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c880039bd4ef8e05f593aab556d82223, disabling compactions & flushes 2024-11-13T07:15:38,933 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5f4b94b50abb029a06bde24412d357dd, disabling compactions & flushes 2024-11-13T07:15:38,933 DEBUG [RS:2;2998b885bdef:37717 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T07:15:38,934 INFO [RS:0;2998b885bdef:44417 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T07:15:38,934 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region SPTestTable,66666664,1731482021829.c880039bd4ef8e05f593aab556d82223. 2024-11-13T07:15:38,934 DEBUG [RS:2;2998b885bdef:37717 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:15:38,934 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region SPTestTable,19999999,1731482021829.5f4b94b50abb029a06bde24412d357dd. 2024-11-13T07:15:38,934 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,66666664,1731482021829.c880039bd4ef8e05f593aab556d82223. 2024-11-13T07:15:38,934 INFO [RS:0;2998b885bdef:44417 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;2998b885bdef:44417. 2024-11-13T07:15:38,934 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,66666664,1731482021829.c880039bd4ef8e05f593aab556d82223. after waiting 0 ms 2024-11-13T07:15:38,934 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,19999999,1731482021829.5f4b94b50abb029a06bde24412d357dd. 2024-11-13T07:15:38,934 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,66666664,1731482021829.c880039bd4ef8e05f593aab556d82223. 2024-11-13T07:15:38,934 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,19999999,1731482021829.5f4b94b50abb029a06bde24412d357dd. after waiting 0 ms 2024-11-13T07:15:38,934 DEBUG [RS:0;2998b885bdef:44417 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T07:15:38,934 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,19999999,1731482021829.5f4b94b50abb029a06bde24412d357dd. 2024-11-13T07:15:38,934 DEBUG [RS:0;2998b885bdef:44417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:15:38,934 INFO [RS:2;2998b885bdef:37717 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T07:15:38,934 INFO [RS:2;2998b885bdef:37717 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T07:15:38,934 INFO [RS:2;2998b885bdef:37717 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T07:15:38,934 INFO [RS:0;2998b885bdef:44417 {}] regionserver.HRegionServer(1321): Waiting on 6 regions to close 2024-11-13T07:15:38,934 INFO [RS:2;2998b885bdef:37717 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T07:15:38,934 DEBUG [RS:0;2998b885bdef:44417 {}] regionserver.HRegionServer(1325): Online Regions={c880039bd4ef8e05f593aab556d82223=SPTestTable,66666664,1731482021829.c880039bd4ef8e05f593aab556d82223., 88b9c24c43eef7986e237d795658427e=SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e., c5c88a6d20592c1b4adcc2269ffce351=SPTestTable,,1731482021829.c5c88a6d20592c1b4adcc2269ffce351., afca0ae186274ac99f7499d163ba2c4e=SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e., 4b6c6542ce1ea1c1d3d96b4831e7c74a=SPTestTable,33333332,1731482021829.4b6c6542ce1ea1c1d3d96b4831e7c74a., feb86c9c548b3a71cf3eef83fbb2adeb=SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb.} 2024-11-13T07:15:38,934 INFO [RS:2;2998b885bdef:37717 {}] regionserver.HRegionServer(1321): Waiting on 5 regions to close 2024-11-13T07:15:38,934 DEBUG [RS:2;2998b885bdef:37717 {}] regionserver.HRegionServer(1325): Online Regions={5f4b94b50abb029a06bde24412d357dd=SPTestTable,19999999,1731482021829.5f4b94b50abb029a06bde24412d357dd., 8ee3029b3e33cc034b84b2d370d7f61d=SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d., 6af43b5a45829fff6c10e65235eba1d3=SPTestTable,7ffffffd,1731482021829.6af43b5a45829fff6c10e65235eba1d3., 1588230740=hbase:meta,,1.1588230740, f70ed29b296742bfa400816128951d24=SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24.} 2024-11-13T07:15:38,934 DEBUG [RS:2;2998b885bdef:37717 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 5f4b94b50abb029a06bde24412d357dd, 6af43b5a45829fff6c10e65235eba1d3, 8ee3029b3e33cc034b84b2d370d7f61d, f70ed29b296742bfa400816128951d24 2024-11-13T07:15:38,934 DEBUG [RS:0;2998b885bdef:44417 {}] regionserver.HRegionServer(1351): Waiting on 4b6c6542ce1ea1c1d3d96b4831e7c74a, 88b9c24c43eef7986e237d795658427e, afca0ae186274ac99f7499d163ba2c4e, c5c88a6d20592c1b4adcc2269ffce351, c880039bd4ef8e05f593aab556d82223, feb86c9c548b3a71cf3eef83fbb2adeb 2024-11-13T07:15:38,935 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T07:15:38,935 INFO [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T07:15:38,935 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T07:15:38,935 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T07:15:38,935 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T07:15:38,935 INFO [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.93 KB heapSize=5.70 KB 2024-11-13T07:15:38,939 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/5f4b94b50abb029a06bde24412d357dd/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-13T07:15:38,939 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/c880039bd4ef8e05f593aab556d82223/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-13T07:15:38,940 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/.tmp/info/cad42c6699f144ac8141fa0f5965f52f is 167, key is SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e./info:regioninfo/1731482086691/Put/seqid=0 2024-11-13T07:15:38,941 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed SPTestTable,66666664,1731482021829.c880039bd4ef8e05f593aab556d82223. 2024-11-13T07:15:38,941 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed SPTestTable,19999999,1731482021829.5f4b94b50abb029a06bde24412d357dd. 2024-11-13T07:15:38,941 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c880039bd4ef8e05f593aab556d82223: Waiting for close lock at 1731482138933Running coprocessor pre-close hooks at 1731482138933Disabling compacts and flushes for region at 1731482138933Disabling writes for close at 1731482138934 (+1 ms)Writing region close event to WAL at 1731482138934Running coprocessor post-close hooks at 1731482138940 (+6 ms)Closed at 1731482138941 (+1 ms) 2024-11-13T07:15:38,941 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5f4b94b50abb029a06bde24412d357dd: Waiting for close lock at 1731482138933Running coprocessor pre-close hooks at 1731482138933Disabling compacts and flushes for region at 1731482138933Disabling writes for close at 1731482138934 (+1 ms)Writing region close event to WAL at 1731482138935 (+1 ms)Running coprocessor post-close hooks at 1731482138940 (+5 ms)Closed at 1731482138941 (+1 ms) 2024-11-13T07:15:38,942 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed SPTestTable,66666664,1731482021829.c880039bd4ef8e05f593aab556d82223. 2024-11-13T07:15:38,942 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed SPTestTable,19999999,1731482021829.5f4b94b50abb029a06bde24412d357dd. 2024-11-13T07:15:38,942 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 88b9c24c43eef7986e237d795658427e, disabling compactions & flushes 2024-11-13T07:15:38,942 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8ee3029b3e33cc034b84b2d370d7f61d, disabling compactions & flushes 2024-11-13T07:15:38,942 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e. 2024-11-13T07:15:38,942 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e. 2024-11-13T07:15:38,942 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d. 2024-11-13T07:15:38,942 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e. after waiting 0 ms 2024-11-13T07:15:38,942 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d. 2024-11-13T07:15:38,942 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e. 2024-11-13T07:15:38,942 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d. after waiting 0 ms 2024-11-13T07:15:38,942 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d. 2024-11-13T07:15:38,947 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/88b9c24c43eef7986e237d795658427e/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-13T07:15:38,948 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e. 2024-11-13T07:15:38,948 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 88b9c24c43eef7986e237d795658427e: Waiting for close lock at 1731482138942Running coprocessor pre-close hooks at 1731482138942Disabling compacts and flushes for region at 1731482138942Disabling writes for close at 1731482138942Writing region close event to WAL at 1731482138943 (+1 ms)Running coprocessor post-close hooks at 1731482138947 (+4 ms)Closed at 1731482138947 2024-11-13T07:15:38,948 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/8ee3029b3e33cc034b84b2d370d7f61d/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T07:15:38,948 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed SPTestTable,4ccccccb,1731482021829.88b9c24c43eef7986e237d795658427e. 2024-11-13T07:15:38,948 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c5c88a6d20592c1b4adcc2269ffce351, disabling compactions & flushes 2024-11-13T07:15:38,948 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region SPTestTable,,1731482021829.c5c88a6d20592c1b4adcc2269ffce351. 2024-11-13T07:15:38,948 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,,1731482021829.c5c88a6d20592c1b4adcc2269ffce351. 2024-11-13T07:15:38,948 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,,1731482021829.c5c88a6d20592c1b4adcc2269ffce351. after waiting 0 ms 2024-11-13T07:15:38,948 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,,1731482021829.c5c88a6d20592c1b4adcc2269ffce351. 2024-11-13T07:15:38,948 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d. 2024-11-13T07:15:38,949 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8ee3029b3e33cc034b84b2d370d7f61d: Waiting for close lock at 1731482138942Running coprocessor pre-close hooks at 1731482138942Disabling compacts and flushes for region at 1731482138942Disabling writes for close at 1731482138942Writing region close event to WAL at 1731482138943 (+1 ms)Running coprocessor post-close hooks at 1731482138948 (+5 ms)Closed at 1731482138948 2024-11-13T07:15:38,949 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed SPTestTable,ccccccc8,1731482021829.8ee3029b3e33cc034b84b2d370d7f61d. 2024-11-13T07:15:38,949 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6af43b5a45829fff6c10e65235eba1d3, disabling compactions & flushes 2024-11-13T07:15:38,949 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region SPTestTable,7ffffffd,1731482021829.6af43b5a45829fff6c10e65235eba1d3. 2024-11-13T07:15:38,949 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,7ffffffd,1731482021829.6af43b5a45829fff6c10e65235eba1d3. 2024-11-13T07:15:38,949 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,7ffffffd,1731482021829.6af43b5a45829fff6c10e65235eba1d3. after waiting 0 ms 2024-11-13T07:15:38,949 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,7ffffffd,1731482021829.6af43b5a45829fff6c10e65235eba1d3. 2024-11-13T07:15:38,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741867_1043 (size=8479) 2024-11-13T07:15:38,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741867_1043 (size=8479) 2024-11-13T07:15:38,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741867_1043 (size=8479) 2024-11-13T07:15:38,951 INFO [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.93 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/.tmp/info/cad42c6699f144ac8141fa0f5965f52f 2024-11-13T07:15:38,959 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/c5c88a6d20592c1b4adcc2269ffce351/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-13T07:15:38,960 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed SPTestTable,,1731482021829.c5c88a6d20592c1b4adcc2269ffce351. 2024-11-13T07:15:38,960 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c5c88a6d20592c1b4adcc2269ffce351: Waiting for close lock at 1731482138948Running coprocessor pre-close hooks at 1731482138948Disabling compacts and flushes for region at 1731482138948Disabling writes for close at 1731482138948Writing region close event to WAL at 1731482138954 (+6 ms)Running coprocessor post-close hooks at 1731482138960 (+6 ms)Closed at 1731482138960 2024-11-13T07:15:38,960 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/6af43b5a45829fff6c10e65235eba1d3/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-13T07:15:38,960 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed SPTestTable,,1731482021829.c5c88a6d20592c1b4adcc2269ffce351. 2024-11-13T07:15:38,960 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing afca0ae186274ac99f7499d163ba2c4e, disabling compactions & flushes 2024-11-13T07:15:38,960 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e. 2024-11-13T07:15:38,960 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e. 2024-11-13T07:15:38,960 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e. after waiting 0 ms 2024-11-13T07:15:38,960 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e. 2024-11-13T07:15:38,961 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed SPTestTable,7ffffffd,1731482021829.6af43b5a45829fff6c10e65235eba1d3. 2024-11-13T07:15:38,961 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6af43b5a45829fff6c10e65235eba1d3: Waiting for close lock at 1731482138949Running coprocessor pre-close hooks at 1731482138949Disabling compacts and flushes for region at 1731482138949Disabling writes for close at 1731482138949Writing region close event to WAL at 1731482138956 (+7 ms)Running coprocessor post-close hooks at 1731482138961 (+5 ms)Closed at 1731482138961 2024-11-13T07:15:38,961 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/.tmp/info/cad42c6699f144ac8141fa0f5965f52f as hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/info/cad42c6699f144ac8141fa0f5965f52f 2024-11-13T07:15:38,961 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed SPTestTable,7ffffffd,1731482021829.6af43b5a45829fff6c10e65235eba1d3. 2024-11-13T07:15:38,961 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f70ed29b296742bfa400816128951d24, disabling compactions & flushes 2024-11-13T07:15:38,961 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24. 2024-11-13T07:15:38,961 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24. 2024-11-13T07:15:38,961 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24. after waiting 0 ms 2024-11-13T07:15:38,961 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24. 2024-11-13T07:15:38,965 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/afca0ae186274ac99f7499d163ba2c4e/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T07:15:38,966 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e. 2024-11-13T07:15:38,966 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for afca0ae186274ac99f7499d163ba2c4e: Waiting for close lock at 1731482138960Running coprocessor pre-close hooks at 1731482138960Disabling compacts and flushes for region at 1731482138960Disabling writes for close at 1731482138960Writing region close event to WAL at 1731482138961 (+1 ms)Running coprocessor post-close hooks at 1731482138966 (+5 ms)Closed at 1731482138966 2024-11-13T07:15:38,966 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed SPTestTable,b333332f,1731482021829.afca0ae186274ac99f7499d163ba2c4e. 2024-11-13T07:15:38,966 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4b6c6542ce1ea1c1d3d96b4831e7c74a, disabling compactions & flushes 2024-11-13T07:15:38,967 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region SPTestTable,33333332,1731482021829.4b6c6542ce1ea1c1d3d96b4831e7c74a. 2024-11-13T07:15:38,967 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,33333332,1731482021829.4b6c6542ce1ea1c1d3d96b4831e7c74a. 2024-11-13T07:15:38,967 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,33333332,1731482021829.4b6c6542ce1ea1c1d3d96b4831e7c74a. after waiting 0 ms 2024-11-13T07:15:38,967 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,33333332,1731482021829.4b6c6542ce1ea1c1d3d96b4831e7c74a. 2024-11-13T07:15:38,968 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/f70ed29b296742bfa400816128951d24/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-13T07:15:38,969 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24. 2024-11-13T07:15:38,969 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f70ed29b296742bfa400816128951d24: Waiting for close lock at 1731482138961Running coprocessor pre-close hooks at 1731482138961Disabling compacts and flushes for region at 1731482138961Disabling writes for close at 1731482138961Writing region close event to WAL at 1731482138964 (+3 ms)Running coprocessor post-close hooks at 1731482138969 (+5 ms)Closed at 1731482138969 2024-11-13T07:15:38,969 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed SPTestTable,99999996,1731482021829.f70ed29b296742bfa400816128951d24. 2024-11-13T07:15:38,970 INFO [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/info/cad42c6699f144ac8141fa0f5965f52f, entries=24, sequenceid=37, filesize=8.3 K 2024-11-13T07:15:38,971 INFO [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.93 KB/2996, heapSize ~4.93 KB/5048, currentSize=0 B/0 for 1588230740 in 36ms, sequenceid=37, compaction requested=false 2024-11-13T07:15:38,971 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/4b6c6542ce1ea1c1d3d96b4831e7c74a/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-13T07:15:38,972 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed SPTestTable,33333332,1731482021829.4b6c6542ce1ea1c1d3d96b4831e7c74a. 2024-11-13T07:15:38,972 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4b6c6542ce1ea1c1d3d96b4831e7c74a: Waiting for close lock at 1731482138966Running coprocessor pre-close hooks at 1731482138966Disabling compacts and flushes for region at 1731482138966Disabling writes for close at 1731482138967 (+1 ms)Writing region close event to WAL at 1731482138967Running coprocessor post-close hooks at 1731482138972 (+5 ms)Closed at 1731482138972 2024-11-13T07:15:38,972 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed SPTestTable,33333332,1731482021829.4b6c6542ce1ea1c1d3d96b4831e7c74a. 2024-11-13T07:15:38,972 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing feb86c9c548b3a71cf3eef83fbb2adeb, disabling compactions & flushes 2024-11-13T07:15:38,972 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb. 2024-11-13T07:15:38,973 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb. 2024-11-13T07:15:38,973 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb. after waiting 0 ms 2024-11-13T07:15:38,973 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb. 2024-11-13T07:15:38,976 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/hbase/meta/1588230740/recovered.edits/40.seqid, newMaxSeqId=40, maxSeqId=28 2024-11-13T07:15:38,976 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/data/default/SPTestTable/feb86c9c548b3a71cf3eef83fbb2adeb/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-13T07:15:38,977 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T07:15:38,977 INFO [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T07:15:38,977 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb. 2024-11-13T07:15:38,977 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731482138935Running coprocessor pre-close hooks at 1731482138935Disabling compacts and flushes for region at 1731482138935Disabling writes for close at 1731482138935Obtaining lock to block concurrent updates at 1731482138935Preparing flush snapshotting stores in 1588230740 at 1731482138935Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2996, getHeapSize=5768, getOffHeapSize=0, getCellsCount=24 at 1731482138936 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731482138936Flushing 1588230740/info: creating writer at 1731482138936Flushing 1588230740/info: appending metadata at 1731482138940 (+4 ms)Flushing 1588230740/info: closing flushed file at 1731482138940Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@215f17e0: reopening flushed file at 1731482138960 (+20 ms)Finished flush of dataSize ~2.93 KB/2996, heapSize ~4.93 KB/5048, currentSize=0 B/0 for 1588230740 in 36ms, sequenceid=37, compaction requested=false at 1731482138971 (+11 ms)Writing region close event to WAL at 1731482138972 (+1 ms)Running coprocessor post-close hooks at 1731482138977 (+5 ms)Closed at 1731482138977 2024-11-13T07:15:38,977 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for feb86c9c548b3a71cf3eef83fbb2adeb: Waiting for close lock at 1731482138972Running coprocessor pre-close hooks at 1731482138972Disabling compacts and flushes for region at 1731482138972Disabling writes for close at 1731482138973 (+1 ms)Writing region close event to WAL at 1731482138973Running coprocessor post-close hooks at 1731482138977 (+4 ms)Closed at 1731482138977 2024-11-13T07:15:38,977 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T07:15:38,977 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed SPTestTable,e6666661,1731482021829.feb86c9c548b3a71cf3eef83fbb2adeb. 2024-11-13T07:15:38,980 INFO [regionserver/2998b885bdef:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T07:15:38,988 INFO [regionserver/2998b885bdef:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T07:15:39,135 INFO [RS:0;2998b885bdef:44417 {}] regionserver.HRegionServer(976): stopping server 2998b885bdef,44417,1731482005920; all regions closed. 2024-11-13T07:15:39,135 INFO [RS:2;2998b885bdef:37717 {}] regionserver.HRegionServer(976): stopping server 2998b885bdef,37717,1731482006937; all regions closed. 2024-11-13T07:15:39,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741835_1011 (size=4083) 2024-11-13T07:15:39,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741835_1011 (size=4083) 2024-11-13T07:15:39,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741835_1011 (size=4083) 2024-11-13T07:15:39,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741859_1035 (size=4408) 2024-11-13T07:15:39,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741859_1035 (size=4408) 2024-11-13T07:15:39,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741859_1035 (size=4408) 2024-11-13T07:15:39,148 DEBUG [RS:2;2998b885bdef:37717 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/oldWALs 2024-11-13T07:15:39,148 INFO [RS:2;2998b885bdef:37717 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2998b885bdef%2C37717%2C1731482006937.meta:.meta(num 1731482065515) 2024-11-13T07:15:39,148 DEBUG [RS:0;2998b885bdef:44417 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/oldWALs 2024-11-13T07:15:39,148 INFO [RS:0;2998b885bdef:44417 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2998b885bdef%2C44417%2C1731482005920:(num 1731482010689) 2024-11-13T07:15:39,148 DEBUG [RS:0;2998b885bdef:44417 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:15:39,148 INFO [RS:0;2998b885bdef:44417 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T07:15:39,149 INFO [RS:0;2998b885bdef:44417 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T07:15:39,149 INFO [RS:0;2998b885bdef:44417 {}] hbase.ChoreService(370): Chore service for: regionserver/2998b885bdef:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T07:15:39,149 INFO [RS:0;2998b885bdef:44417 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T07:15:39,149 INFO [RS:0;2998b885bdef:44417 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T07:15:39,149 INFO [regionserver/2998b885bdef:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T07:15:39,149 INFO [RS:0;2998b885bdef:44417 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T07:15:39,149 INFO [RS:0;2998b885bdef:44417 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T07:15:39,150 INFO [RS:0;2998b885bdef:44417 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44417 2024-11-13T07:15:39,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741834_1010 (size=2947) 2024-11-13T07:15:39,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741834_1010 (size=2947) 2024-11-13T07:15:39,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741834_1010 (size=2947) 2024-11-13T07:15:39,154 DEBUG [RS:2;2998b885bdef:37717 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/oldWALs 2024-11-13T07:15:39,155 INFO [RS:2;2998b885bdef:37717 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2998b885bdef%2C37717%2C1731482006937:(num 1731482010625) 2024-11-13T07:15:39,155 DEBUG [RS:2;2998b885bdef:37717 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:15:39,155 INFO [RS:2;2998b885bdef:37717 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T07:15:39,155 INFO [RS:2;2998b885bdef:37717 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T07:15:39,155 INFO [RS:2;2998b885bdef:37717 {}] hbase.ChoreService(370): Chore service for: regionserver/2998b885bdef:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T07:15:39,155 INFO [RS:2;2998b885bdef:37717 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T07:15:39,155 INFO [regionserver/2998b885bdef:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T07:15:39,155 INFO [RS:2;2998b885bdef:37717 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37717 2024-11-13T07:15:39,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T07:15:39,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101330aa3af0001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2998b885bdef,44417,1731482005920 2024-11-13T07:15:39,157 INFO [RS:0;2998b885bdef:44417 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T07:15:39,168 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37717-0x101330aa3af0003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2998b885bdef,37717,1731482006937 2024-11-13T07:15:39,168 INFO [RS:2;2998b885bdef:37717 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T07:15:39,179 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2998b885bdef,37717,1731482006937] 2024-11-13T07:15:39,200 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2998b885bdef,37717,1731482006937 already deleted, retry=false 2024-11-13T07:15:39,200 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2998b885bdef,37717,1731482006937 expired; onlineServers=1 2024-11-13T07:15:39,200 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2998b885bdef,44417,1731482005920] 2024-11-13T07:15:39,210 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2998b885bdef,44417,1731482005920 already deleted, retry=false 2024-11-13T07:15:39,210 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2998b885bdef,44417,1731482005920 expired; onlineServers=0 2024-11-13T07:15:39,210 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '2998b885bdef,46239,1731482004610' ***** 2024-11-13T07:15:39,211 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T07:15:39,211 INFO [M:0;2998b885bdef:46239 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T07:15:39,211 INFO [M:0;2998b885bdef:46239 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T07:15:39,211 DEBUG [M:0;2998b885bdef:46239 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T07:15:39,211 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T07:15:39,211 DEBUG [M:0;2998b885bdef:46239 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T07:15:39,211 DEBUG [master/2998b885bdef:0:becomeActiveMaster-HFileCleaner.large.0-1731482009881 {}] cleaner.HFileCleaner(306): Exit Thread[master/2998b885bdef:0:becomeActiveMaster-HFileCleaner.large.0-1731482009881,5,FailOnTimeoutGroup] 2024-11-13T07:15:39,211 DEBUG [master/2998b885bdef:0:becomeActiveMaster-HFileCleaner.small.0-1731482009902 {}] cleaner.HFileCleaner(306): Exit Thread[master/2998b885bdef:0:becomeActiveMaster-HFileCleaner.small.0-1731482009902,5,FailOnTimeoutGroup] 2024-11-13T07:15:39,211 INFO [M:0;2998b885bdef:46239 {}] hbase.ChoreService(370): Chore service for: master/2998b885bdef:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T07:15:39,211 INFO [M:0;2998b885bdef:46239 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T07:15:39,211 DEBUG [M:0;2998b885bdef:46239 {}] master.HMaster(1795): Stopping service threads 2024-11-13T07:15:39,212 INFO [M:0;2998b885bdef:46239 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T07:15:39,212 INFO [M:0;2998b885bdef:46239 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T07:15:39,212 ERROR [M:0;2998b885bdef:46239 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1506214772) connection to localhost/127.0.0.1:41763 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:41763,5,PEWorkerGroup] 2024-11-13T07:15:39,212 INFO [M:0;2998b885bdef:46239 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T07:15:39,213 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T07:15:39,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T07:15:39,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:15:39,221 DEBUG [M:0;2998b885bdef:46239 {}] zookeeper.ZKUtil(347): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T07:15:39,221 WARN [M:0;2998b885bdef:46239 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T07:15:39,222 INFO [M:0;2998b885bdef:46239 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/.lastflushedseqids 2024-11-13T07:15:39,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741868_1044 (size=668) 2024-11-13T07:15:39,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741868_1044 (size=668) 2024-11-13T07:15:39,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741868_1044 (size=668) 2024-11-13T07:15:39,236 INFO [M:0;2998b885bdef:46239 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T07:15:39,236 INFO [M:0;2998b885bdef:46239 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T07:15:39,236 DEBUG [M:0;2998b885bdef:46239 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T07:15:39,236 INFO [M:0;2998b885bdef:46239 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T07:15:39,236 DEBUG [M:0;2998b885bdef:46239 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T07:15:39,236 DEBUG [M:0;2998b885bdef:46239 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T07:15:39,236 DEBUG [M:0;2998b885bdef:46239 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T07:15:39,236 INFO [M:0;2998b885bdef:46239 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=203.29 KB heapSize=252.43 KB 2024-11-13T07:15:39,251 DEBUG [M:0;2998b885bdef:46239 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/314aa8ac938c4eefbf17d62533b00ba4 is 82, key is hbase:meta,,1/info:regioninfo/1731482066132/Put/seqid=0 2024-11-13T07:15:39,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741869_1045 (size=6071) 2024-11-13T07:15:39,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741869_1045 (size=6071) 2024-11-13T07:15:39,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741869_1045 (size=6071) 2024-11-13T07:15:39,258 INFO [M:0;2998b885bdef:46239 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/314aa8ac938c4eefbf17d62533b00ba4 2024-11-13T07:15:39,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101330aa3af0001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T07:15:39,279 INFO [RS:0;2998b885bdef:44417 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T07:15:39,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44417-0x101330aa3af0001, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T07:15:39,279 INFO [RS:0;2998b885bdef:44417 {}] regionserver.HRegionServer(1031): Exiting; stopping=2998b885bdef,44417,1731482005920; zookeeper connection closed. 2024-11-13T07:15:39,279 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6083684c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6083684c 2024-11-13T07:15:39,281 DEBUG [M:0;2998b885bdef:46239 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1292f322c66643b48b5d82675d5f2f17 is 1244, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731482033028/Put/seqid=0 2024-11-13T07:15:39,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741870_1046 (size=9043) 2024-11-13T07:15:39,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741870_1046 (size=9043) 2024-11-13T07:15:39,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741870_1046 (size=9043) 2024-11-13T07:15:39,289 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37717-0x101330aa3af0003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T07:15:39,289 INFO [RS:2;2998b885bdef:37717 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T07:15:39,289 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37717-0x101330aa3af0003, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T07:15:39,289 INFO [RS:2;2998b885bdef:37717 {}] regionserver.HRegionServer(1031): Exiting; stopping=2998b885bdef,37717,1731482006937; zookeeper connection closed. 2024-11-13T07:15:39,290 INFO [M:0;2998b885bdef:46239 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=202.06 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1292f322c66643b48b5d82675d5f2f17 2024-11-13T07:15:39,290 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@58c4c1b9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@58c4c1b9 2024-11-13T07:15:39,290 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-13T07:15:39,298 INFO [M:0;2998b885bdef:46239 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1292f322c66643b48b5d82675d5f2f17 2024-11-13T07:15:39,313 DEBUG [M:0;2998b885bdef:46239 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/237c0cdf0ff642a993eb61db1f3ef909 is 69, key is 2998b885bdef,37717,1731482006937/rs:state/1731482009980/Put/seqid=0 2024-11-13T07:15:39,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741871_1047 (size=5441) 2024-11-13T07:15:39,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741871_1047 (size=5441) 2024-11-13T07:15:39,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741871_1047 (size=5441) 2024-11-13T07:15:39,321 INFO [M:0;2998b885bdef:46239 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/237c0cdf0ff642a993eb61db1f3ef909 2024-11-13T07:15:39,327 INFO [M:0;2998b885bdef:46239 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 237c0cdf0ff642a993eb61db1f3ef909 2024-11-13T07:15:39,328 DEBUG [M:0;2998b885bdef:46239 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/314aa8ac938c4eefbf17d62533b00ba4 as hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/314aa8ac938c4eefbf17d62533b00ba4 2024-11-13T07:15:39,335 INFO [M:0;2998b885bdef:46239 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/314aa8ac938c4eefbf17d62533b00ba4, entries=14, sequenceid=535, filesize=5.9 K 2024-11-13T07:15:39,337 DEBUG [M:0;2998b885bdef:46239 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1292f322c66643b48b5d82675d5f2f17 as hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1292f322c66643b48b5d82675d5f2f17 2024-11-13T07:15:39,344 INFO [M:0;2998b885bdef:46239 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1292f322c66643b48b5d82675d5f2f17 2024-11-13T07:15:39,344 INFO [M:0;2998b885bdef:46239 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1292f322c66643b48b5d82675d5f2f17, entries=58, sequenceid=535, filesize=8.8 K 2024-11-13T07:15:39,345 DEBUG [M:0;2998b885bdef:46239 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/237c0cdf0ff642a993eb61db1f3ef909 as hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/237c0cdf0ff642a993eb61db1f3ef909 2024-11-13T07:15:39,351 INFO [M:0;2998b885bdef:46239 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 237c0cdf0ff642a993eb61db1f3ef909 2024-11-13T07:15:39,352 INFO [M:0;2998b885bdef:46239 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41763/user/jenkins/test-data/4a106bbe-3476-ce9b-d946-05ffae295bd9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/237c0cdf0ff642a993eb61db1f3ef909, entries=3, sequenceid=535, filesize=5.3 K 2024-11-13T07:15:39,353 INFO [M:0;2998b885bdef:46239 {}] regionserver.HRegion(3140): Finished flush of dataSize ~203.29 KB/208167, heapSize ~252.13 KB/258184, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 117ms, sequenceid=535, compaction requested=false 2024-11-13T07:15:39,357 INFO [M:0;2998b885bdef:46239 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T07:15:39,357 DEBUG [M:0;2998b885bdef:46239 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731482139236Disabling compacts and flushes for region at 1731482139236Disabling writes for close at 1731482139236Obtaining lock to block concurrent updates at 1731482139236Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731482139236Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=208167, getHeapSize=258424, getOffHeapSize=0, getCellsCount=652 at 1731482139237 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731482139237Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731482139237Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731482139251 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731482139251Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731482139264 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731482139281 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731482139281Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731482139298 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731482139312 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731482139312Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e40eed3: reopening flushed file at 1731482139327 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70277777: reopening flushed file at 1731482139335 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@189326c4: reopening flushed file at 1731482139344 (+9 ms)Finished flush of dataSize ~203.29 KB/208167, heapSize ~252.13 KB/258184, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 117ms, sequenceid=535, compaction requested=false at 1731482139353 (+9 ms)Writing region close event to WAL at 1731482139357 (+4 ms)Closed at 1731482139357 2024-11-13T07:15:39,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40257 is added to blk_1073741830_1006 (size=243829) 2024-11-13T07:15:39,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38027 is added to blk_1073741830_1006 (size=243829) 2024-11-13T07:15:39,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38559 is added to blk_1073741830_1006 (size=243829) 2024-11-13T07:15:39,361 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T07:15:39,361 INFO [M:0;2998b885bdef:46239 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T07:15:39,361 INFO [M:0;2998b885bdef:46239 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46239 2024-11-13T07:15:39,361 INFO [M:0;2998b885bdef:46239 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T07:15:39,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T07:15:39,474 INFO [M:0;2998b885bdef:46239 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T07:15:39,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46239-0x101330aa3af0000, quorum=127.0.0.1:56395, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T07:15:39,480 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7e7c21d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T07:15:39,492 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@30cac06d{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T07:15:39,492 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T07:15:39,492 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3abb77fb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T07:15:39,492 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@55fcaba0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/hadoop.log.dir/,STOPPED} 2024-11-13T07:15:39,496 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T07:15:39,496 WARN [BP-759209407-172.17.0.3-1731481998077 heartbeating to localhost/127.0.0.1:41763 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T07:15:39,496 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T07:15:39,496 WARN [BP-759209407-172.17.0.3-1731481998077 heartbeating to localhost/127.0.0.1:41763 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-759209407-172.17.0.3-1731481998077 (Datanode Uuid 8e69b974-87ae-475f-8e51-ecf56375f722) service to localhost/127.0.0.1:41763 2024-11-13T07:15:39,497 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/cluster_fa0907df-988e-740e-d6e3-0a6b8ca54eeb/data/data5/current/BP-759209407-172.17.0.3-1731481998077 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T07:15:39,497 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/cluster_fa0907df-988e-740e-d6e3-0a6b8ca54eeb/data/data6/current/BP-759209407-172.17.0.3-1731481998077 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T07:15:39,498 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T07:15:39,500 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@9d2632f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T07:15:39,500 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c8e80d{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T07:15:39,500 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T07:15:39,500 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7a2b6a73{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T07:15:39,500 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@cae7855{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/hadoop.log.dir/,STOPPED} 2024-11-13T07:15:39,501 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T07:15:39,501 WARN [BP-759209407-172.17.0.3-1731481998077 heartbeating to localhost/127.0.0.1:41763 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T07:15:39,502 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T07:15:39,502 WARN [BP-759209407-172.17.0.3-1731481998077 heartbeating to localhost/127.0.0.1:41763 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-759209407-172.17.0.3-1731481998077 (Datanode Uuid a8997e61-4fa8-4cfe-b858-24f19c11cc2b) service to localhost/127.0.0.1:41763 2024-11-13T07:15:39,502 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/cluster_fa0907df-988e-740e-d6e3-0a6b8ca54eeb/data/data3/current/BP-759209407-172.17.0.3-1731481998077 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T07:15:39,502 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/cluster_fa0907df-988e-740e-d6e3-0a6b8ca54eeb/data/data4/current/BP-759209407-172.17.0.3-1731481998077 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T07:15:39,503 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T07:15:39,504 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1722ba9a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T07:15:39,505 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@57faae2b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T07:15:39,505 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T07:15:39,505 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@55b4afc5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T07:15:39,505 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@225fb320{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/hadoop.log.dir/,STOPPED} 2024-11-13T07:15:39,506 WARN [BP-759209407-172.17.0.3-1731481998077 heartbeating to localhost/127.0.0.1:41763 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T07:15:39,506 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T07:15:39,506 WARN [BP-759209407-172.17.0.3-1731481998077 heartbeating to localhost/127.0.0.1:41763 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-759209407-172.17.0.3-1731481998077 (Datanode Uuid 813ce77b-5e63-4202-89e0-15afd447938b) service to localhost/127.0.0.1:41763 2024-11-13T07:15:39,506 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T07:15:39,507 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/cluster_fa0907df-988e-740e-d6e3-0a6b8ca54eeb/data/data1/current/BP-759209407-172.17.0.3-1731481998077 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T07:15:39,507 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/cluster_fa0907df-988e-740e-d6e3-0a6b8ca54eeb/data/data2/current/BP-759209407-172.17.0.3-1731481998077 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T07:15:39,507 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T07:15:39,516 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7f9aa94c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T07:15:39,516 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13717375{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T07:15:39,517 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T07:15:39,517 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@39a64764{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T07:15:39,517 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7384c3bf{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/hadoop.log.dir/,STOPPED} 2024-11-13T07:15:39,526 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T07:15:39,579 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-13T07:15:39,588 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.procedure.TestSnapshotProcedureRSCrashes#testRegionServerCrashWhileTakingSnapshot Thread=90 (was 11) Potentially hanging thread: IPC Client (1506214772) connection to localhost/127.0.0.1:41763 from jenkins.hfs.0 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-7-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/2998b885bdef:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/2998b885bdef:0.Chore.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41763 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Abort regionserver monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:41763 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: regionserver/2998b885bdef:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: region-location-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/2998b885bdef:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_META-regionserver/2998b885bdef:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:41763 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41763 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-6-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-6-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: nioEventLoopGroup-7-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-7-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/2998b885bdef:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-6-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@3176dd7a java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SSL Certificates Store Monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:41763 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/2998b885bdef:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41763 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: BootstrapNodeManager java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1506214772) connection to localhost/127.0.0.1:41763 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1506214772) connection to localhost/127.0.0.1:41763 from jenkins.hfs.2 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1506214772) connection to localhost/127.0.0.1:41763 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41763 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/2998b885bdef:0.logRoller java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:180) Potentially hanging thread: master/2998b885bdef:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=440 (was 285) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=450 (was 844), ProcessCount=11 (was 11), AvailableMemoryMB=4005 (was 5608) 2024-11-13T07:15:39,593 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.procedure.TestSnapshotProcedureRSCrashes#testRegionServerCrashWhileVerifyingSnapshot Thread=91, OpenFileDescriptor=440, MaxFileDescriptor=1048576, SystemLoadAverage=450, ProcessCount=11, AvailableMemoryMB=4005 2024-11-13T07:15:39,606 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/hadoop.log.dir so I do NOT create it in target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa 2024-11-13T07:15:39,606 WARN [Time-limited test {}] hbase.HBaseTestingUtil(405): hadoop.log.dir property value differs in configuration and system: Configuration=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/../logs while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/hadoop.log.dir Erasing configuration value by system value. 2024-11-13T07:15:39,606 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/hadoop.tmp.dir so I do NOT create it in target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa 2024-11-13T07:15:39,607 WARN [Time-limited test {}] hbase.HBaseTestingUtil(405): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/dff4576e-fedc-90af-def5-4fc14459bc36/hadoop.tmp.dir Erasing configuration value by system value. 2024-11-13T07:15:39,607 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa 2024-11-13T07:15:39,607 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T07:15:39,607 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/cluster_b80e0f79-8e0b-0df0-031b-f8964be40503, deleteOnExit=true 2024-11-13T07:15:39,607 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T07:15:39,607 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/test.cache.data in system properties and HBase conf 2024-11-13T07:15:39,607 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T07:15:39,607 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/hadoop.log.dir in system properties and HBase conf 2024-11-13T07:15:39,607 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T07:15:39,607 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T07:15:39,607 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T07:15:39,607 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T07:15:39,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T07:15:39,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T07:15:39,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T07:15:39,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T07:15:39,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T07:15:39,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T07:15:39,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T07:15:39,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T07:15:39,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T07:15:39,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/nfs.dump.dir in system properties and HBase conf 2024-11-13T07:15:39,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/java.io.tmpdir in system properties and HBase conf 2024-11-13T07:15:39,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T07:15:39,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T07:15:39,609 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T07:15:40,152 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T07:15:40,161 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T07:15:40,163 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T07:15:40,163 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T07:15:40,163 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T07:15:40,164 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T07:15:40,165 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7b359d2b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/hadoop.log.dir/,AVAILABLE} 2024-11-13T07:15:40,166 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3368b9d6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T07:15:40,288 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2ac7a495{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/java.io.tmpdir/jetty-localhost-38317-hadoop-hdfs-3_4_1-tests_jar-_-any-900366306871381229/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T07:15:40,289 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@29729763{HTTP/1.1, (http/1.1)}{localhost:38317} 2024-11-13T07:15:40,289 INFO [Time-limited test {}] server.Server(415): Started @145452ms 2024-11-13T07:15:40,802 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T07:15:40,814 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T07:15:40,815 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T07:15:40,815 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T07:15:40,816 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T07:15:40,816 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5aa85305{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/hadoop.log.dir/,AVAILABLE} 2024-11-13T07:15:40,817 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@735f3901{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T07:15:40,941 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@bf8095d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/java.io.tmpdir/jetty-localhost-44955-hadoop-hdfs-3_4_1-tests_jar-_-any-17934801700310828880/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T07:15:40,941 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@14ef5eb2{HTTP/1.1, (http/1.1)}{localhost:44955} 2024-11-13T07:15:40,941 INFO [Time-limited test {}] server.Server(415): Started @146105ms 2024-11-13T07:15:40,943 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T07:15:41,009 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T07:15:41,021 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T07:15:41,028 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T07:15:41,028 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T07:15:41,028 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T07:15:41,029 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4de59280{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/hadoop.log.dir/,AVAILABLE} 2024-11-13T07:15:41,029 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70d5749e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T07:15:41,159 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@9234fcc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/java.io.tmpdir/jetty-localhost-33923-hadoop-hdfs-3_4_1-tests_jar-_-any-9865750506628130122/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T07:15:41,160 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@a50cd13{HTTP/1.1, (http/1.1)}{localhost:33923} 2024-11-13T07:15:41,160 INFO [Time-limited test {}] server.Server(415): Started @146323ms 2024-11-13T07:15:41,162 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T07:15:41,250 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T07:15:41,256 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T07:15:41,260 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T07:15:41,260 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T07:15:41,260 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T07:15:41,265 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@74ede052{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/hadoop.log.dir/,AVAILABLE} 2024-11-13T07:15:41,265 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@549ff785{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T07:15:41,416 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@27eeff62{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/java.io.tmpdir/jetty-localhost-40885-hadoop-hdfs-3_4_1-tests_jar-_-any-13750941876540153579/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T07:15:41,417 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7cfe3ae0{HTTP/1.1, (http/1.1)}{localhost:40885} 2024-11-13T07:15:41,417 INFO [Time-limited test {}] server.Server(415): Started @146580ms 2024-11-13T07:15:41,418 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T07:15:42,845 WARN [Thread-743 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/cluster_b80e0f79-8e0b-0df0-031b-f8964be40503/data/data1/current/BP-2139882861-172.17.0.3-1731482139625/current, will proceed with Du for space computation calculation, 2024-11-13T07:15:42,845 WARN [Thread-744 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/cluster_b80e0f79-8e0b-0df0-031b-f8964be40503/data/data2/current/BP-2139882861-172.17.0.3-1731482139625/current, will proceed with Du for space computation calculation, 2024-11-13T07:15:42,880 WARN [Thread-684 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T07:15:42,883 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x883e3c48d5346f52 with lease ID 0xeb519a90a920db6d: Processing first storage report for DS-3008da15-b38a-4db4-8f70-4870724e90c0 from datanode DatanodeRegistration(127.0.0.1:46793, datanodeUuid=82436f89-bc06-4064-adfb-71c4761c3e81, infoPort=41379, infoSecurePort=0, ipcPort=45067, storageInfo=lv=-57;cid=testClusterID;nsid=1671332803;c=1731482139625) 2024-11-13T07:15:42,884 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x883e3c48d5346f52 with lease ID 0xeb519a90a920db6d: from storage DS-3008da15-b38a-4db4-8f70-4870724e90c0 node DatanodeRegistration(127.0.0.1:46793, datanodeUuid=82436f89-bc06-4064-adfb-71c4761c3e81, infoPort=41379, infoSecurePort=0, ipcPort=45067, storageInfo=lv=-57;cid=testClusterID;nsid=1671332803;c=1731482139625), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T07:15:42,884 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x883e3c48d5346f52 with lease ID 0xeb519a90a920db6d: Processing first storage report for DS-e59aecdd-e265-48e5-8157-f6e54ff50f86 from datanode DatanodeRegistration(127.0.0.1:46793, datanodeUuid=82436f89-bc06-4064-adfb-71c4761c3e81, infoPort=41379, infoSecurePort=0, ipcPort=45067, storageInfo=lv=-57;cid=testClusterID;nsid=1671332803;c=1731482139625) 2024-11-13T07:15:42,884 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x883e3c48d5346f52 with lease ID 0xeb519a90a920db6d: from storage DS-e59aecdd-e265-48e5-8157-f6e54ff50f86 node DatanodeRegistration(127.0.0.1:46793, datanodeUuid=82436f89-bc06-4064-adfb-71c4761c3e81, infoPort=41379, infoSecurePort=0, ipcPort=45067, storageInfo=lv=-57;cid=testClusterID;nsid=1671332803;c=1731482139625), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T07:15:43,232 WARN [Thread-755 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/cluster_b80e0f79-8e0b-0df0-031b-f8964be40503/data/data3/current/BP-2139882861-172.17.0.3-1731482139625/current, will proceed with Du for space computation calculation, 2024-11-13T07:15:43,237 WARN [Thread-756 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/cluster_b80e0f79-8e0b-0df0-031b-f8964be40503/data/data4/current/BP-2139882861-172.17.0.3-1731482139625/current, will proceed with Du for space computation calculation, 2024-11-13T07:15:43,253 WARN [Thread-707 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T07:15:43,255 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9fc744c5c197be9c with lease ID 0xeb519a90a920db6e: Processing first storage report for DS-e60c6474-3785-4b26-bb2a-dccf30dc623d from datanode DatanodeRegistration(127.0.0.1:45017, datanodeUuid=8d67d94d-4516-49da-880d-0d1102fd17e3, infoPort=32917, infoSecurePort=0, ipcPort=37599, storageInfo=lv=-57;cid=testClusterID;nsid=1671332803;c=1731482139625) 2024-11-13T07:15:43,255 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9fc744c5c197be9c with lease ID 0xeb519a90a920db6e: from storage DS-e60c6474-3785-4b26-bb2a-dccf30dc623d node DatanodeRegistration(127.0.0.1:45017, datanodeUuid=8d67d94d-4516-49da-880d-0d1102fd17e3, infoPort=32917, infoSecurePort=0, ipcPort=37599, storageInfo=lv=-57;cid=testClusterID;nsid=1671332803;c=1731482139625), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T07:15:43,255 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9fc744c5c197be9c with lease ID 0xeb519a90a920db6e: Processing first storage report for DS-2e05d3d4-39e1-40a7-bb21-3ac68c6fdc91 from datanode DatanodeRegistration(127.0.0.1:45017, datanodeUuid=8d67d94d-4516-49da-880d-0d1102fd17e3, infoPort=32917, infoSecurePort=0, ipcPort=37599, storageInfo=lv=-57;cid=testClusterID;nsid=1671332803;c=1731482139625) 2024-11-13T07:15:43,256 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9fc744c5c197be9c with lease ID 0xeb519a90a920db6e: from storage DS-2e05d3d4-39e1-40a7-bb21-3ac68c6fdc91 node DatanodeRegistration(127.0.0.1:45017, datanodeUuid=8d67d94d-4516-49da-880d-0d1102fd17e3, infoPort=32917, infoSecurePort=0, ipcPort=37599, storageInfo=lv=-57;cid=testClusterID;nsid=1671332803;c=1731482139625), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T07:15:43,359 WARN [Thread-766 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/cluster_b80e0f79-8e0b-0df0-031b-f8964be40503/data/data5/current/BP-2139882861-172.17.0.3-1731482139625/current, will proceed with Du for space computation calculation, 2024-11-13T07:15:43,359 WARN [Thread-767 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/cluster_b80e0f79-8e0b-0df0-031b-f8964be40503/data/data6/current/BP-2139882861-172.17.0.3-1731482139625/current, will proceed with Du for space computation calculation, 2024-11-13T07:15:43,385 WARN [Thread-729 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T07:15:43,390 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4a2a1d5def7babaf with lease ID 0xeb519a90a920db6f: Processing first storage report for DS-e9be3a87-9173-46cd-97c2-458b82422043 from datanode DatanodeRegistration(127.0.0.1:43743, datanodeUuid=5dc6eee8-bc83-4b46-861c-9804935efd7d, infoPort=35119, infoSecurePort=0, ipcPort=44679, storageInfo=lv=-57;cid=testClusterID;nsid=1671332803;c=1731482139625) 2024-11-13T07:15:43,391 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4a2a1d5def7babaf with lease ID 0xeb519a90a920db6f: from storage DS-e9be3a87-9173-46cd-97c2-458b82422043 node DatanodeRegistration(127.0.0.1:43743, datanodeUuid=5dc6eee8-bc83-4b46-861c-9804935efd7d, infoPort=35119, infoSecurePort=0, ipcPort=44679, storageInfo=lv=-57;cid=testClusterID;nsid=1671332803;c=1731482139625), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T07:15:43,391 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4a2a1d5def7babaf with lease ID 0xeb519a90a920db6f: Processing first storage report for DS-e3ec9889-a458-4148-b144-e92cc8b781d3 from datanode DatanodeRegistration(127.0.0.1:43743, datanodeUuid=5dc6eee8-bc83-4b46-861c-9804935efd7d, infoPort=35119, infoSecurePort=0, ipcPort=44679, storageInfo=lv=-57;cid=testClusterID;nsid=1671332803;c=1731482139625) 2024-11-13T07:15:43,391 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4a2a1d5def7babaf with lease ID 0xeb519a90a920db6f: from storage DS-e3ec9889-a458-4148-b144-e92cc8b781d3 node DatanodeRegistration(127.0.0.1:43743, datanodeUuid=5dc6eee8-bc83-4b46-861c-9804935efd7d, infoPort=35119, infoSecurePort=0, ipcPort=44679, storageInfo=lv=-57;cid=testClusterID;nsid=1671332803;c=1731482139625), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T07:15:43,485 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa 2024-11-13T07:15:43,490 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/cluster_b80e0f79-8e0b-0df0-031b-f8964be40503/zookeeper_0, clientPort=63253, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/cluster_b80e0f79-8e0b-0df0-031b-f8964be40503/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/cluster_b80e0f79-8e0b-0df0-031b-f8964be40503/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T07:15:43,491 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63253 2024-11-13T07:15:43,491 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:15:43,494 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:15:43,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741825_1001 (size=7) 2024-11-13T07:15:43,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741825_1001 (size=7) 2024-11-13T07:15:43,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741825_1001 (size=7) 2024-11-13T07:15:43,513 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca with version=8 2024-11-13T07:15:43,514 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/hbase-staging 2024-11-13T07:15:43,517 INFO [Time-limited test {}] client.ConnectionUtils(128): master/2998b885bdef:0 server-side Connection retries=45 2024-11-13T07:15:43,517 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T07:15:43,517 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T07:15:43,518 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T07:15:43,518 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T07:15:43,518 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T07:15:43,518 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T07:15:43,518 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T07:15:43,519 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44607 2024-11-13T07:15:43,521 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44607 connecting to ZooKeeper ensemble=127.0.0.1:63253 2024-11-13T07:15:43,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:446070x0, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T07:15:43,611 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44607-0x101330cc6ab0000 connected 2024-11-13T07:15:43,781 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:15:43,783 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:15:43,786 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T07:15:43,786 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca, hbase.cluster.distributed=false 2024-11-13T07:15:43,788 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T07:15:43,788 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44607 2024-11-13T07:15:43,789 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44607 2024-11-13T07:15:43,789 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44607 2024-11-13T07:15:43,790 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44607 2024-11-13T07:15:43,790 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44607 2024-11-13T07:15:43,809 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2998b885bdef:0 server-side Connection retries=45 2024-11-13T07:15:43,809 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T07:15:43,809 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T07:15:43,809 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T07:15:43,809 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T07:15:43,809 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T07:15:43,809 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T07:15:43,809 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T07:15:43,810 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41365 2024-11-13T07:15:43,811 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41365 connecting to ZooKeeper ensemble=127.0.0.1:63253 2024-11-13T07:15:43,812 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:15:43,814 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:15:43,861 DEBUG [pool-309-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:413650x0, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T07:15:43,861 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41365-0x101330cc6ab0001, quorum=127.0.0.1:63253, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T07:15:43,861 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41365-0x101330cc6ab0001 connected 2024-11-13T07:15:43,862 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T07:15:43,862 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T07:15:43,863 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41365-0x101330cc6ab0001, quorum=127.0.0.1:63253, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T07:15:43,864 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41365-0x101330cc6ab0001, quorum=127.0.0.1:63253, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T07:15:43,864 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41365 2024-11-13T07:15:43,864 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41365 2024-11-13T07:15:43,865 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41365 2024-11-13T07:15:43,865 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41365 2024-11-13T07:15:43,865 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41365 2024-11-13T07:15:43,882 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2998b885bdef:0 server-side Connection retries=45 2024-11-13T07:15:43,883 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T07:15:43,883 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T07:15:43,883 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T07:15:43,883 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T07:15:43,883 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T07:15:43,883 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T07:15:43,883 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T07:15:43,884 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39195 2024-11-13T07:15:43,885 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39195 connecting to ZooKeeper ensemble=127.0.0.1:63253 2024-11-13T07:15:43,886 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:15:43,887 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:15:43,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:391950x0, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T07:15:43,939 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39195-0x101330cc6ab0002 connected 2024-11-13T07:15:43,939 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39195-0x101330cc6ab0002, quorum=127.0.0.1:63253, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T07:15:43,940 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T07:15:43,941 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T07:15:43,942 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39195-0x101330cc6ab0002, quorum=127.0.0.1:63253, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T07:15:43,944 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39195-0x101330cc6ab0002, quorum=127.0.0.1:63253, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T07:15:43,945 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39195 2024-11-13T07:15:43,947 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39195 2024-11-13T07:15:43,947 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T07:15:43,947 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39195 2024-11-13T07:15:43,948 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39195 2024-11-13T07:15:43,949 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39195 2024-11-13T07:15:43,963 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T07:15:43,963 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T07:15:43,970 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2998b885bdef:0 server-side Connection retries=45 2024-11-13T07:15:43,970 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T07:15:43,971 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T07:15:43,971 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T07:15:43,971 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T07:15:43,971 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T07:15:43,971 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T07:15:43,971 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T07:15:43,972 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36543 2024-11-13T07:15:43,974 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36543 connecting to ZooKeeper ensemble=127.0.0.1:63253 2024-11-13T07:15:43,975 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:15:43,977 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:15:44,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:365430x0, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T07:15:44,002 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36543-0x101330cc6ab0003 connected 2024-11-13T07:15:44,002 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36543-0x101330cc6ab0003, quorum=127.0.0.1:63253, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T07:15:44,003 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T07:15:44,008 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T07:15:44,009 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36543-0x101330cc6ab0003, quorum=127.0.0.1:63253, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T07:15:44,011 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36543-0x101330cc6ab0003, quorum=127.0.0.1:63253, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T07:15:44,012 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36543 2024-11-13T07:15:44,012 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36543 2024-11-13T07:15:44,012 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36543 2024-11-13T07:15:44,013 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36543 2024-11-13T07:15:44,013 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36543 2024-11-13T07:15:44,031 DEBUG [M:0;2998b885bdef:44607 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;2998b885bdef:44607 2024-11-13T07:15:44,031 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/2998b885bdef,44607,1731482143517 2024-11-13T07:15:44,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T07:15:44,097 DEBUG [pool-309-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41365-0x101330cc6ab0001, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T07:15:44,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39195-0x101330cc6ab0002, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T07:15:44,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36543-0x101330cc6ab0003, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T07:15:44,099 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/2998b885bdef,44607,1731482143517 2024-11-13T07:15:44,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:15:44,139 DEBUG [pool-309-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41365-0x101330cc6ab0001, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T07:15:44,139 DEBUG [pool-309-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41365-0x101330cc6ab0001, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:15:44,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39195-0x101330cc6ab0002, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T07:15:44,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39195-0x101330cc6ab0002, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:15:44,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36543-0x101330cc6ab0003, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T07:15:44,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36543-0x101330cc6ab0003, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:15:44,141 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T07:15:44,142 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/2998b885bdef,44607,1731482143517 from backup master directory 2024-11-13T07:15:44,159 DEBUG [pool-309-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41365-0x101330cc6ab0001, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T07:15:44,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39195-0x101330cc6ab0002, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T07:15:44,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/2998b885bdef,44607,1731482143517 2024-11-13T07:15:44,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36543-0x101330cc6ab0003, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T07:15:44,160 WARN [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T07:15:44,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T07:15:44,160 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=2998b885bdef,44607,1731482143517 2024-11-13T07:15:44,166 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/hbase.id] with ID: d9509229-0f8c-4541-803c-0e13c5ef5f10 2024-11-13T07:15:44,166 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/.tmp/hbase.id 2024-11-13T07:15:44,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741826_1002 (size=42) 2024-11-13T07:15:44,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741826_1002 (size=42) 2024-11-13T07:15:44,175 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/.tmp/hbase.id]:[hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/hbase.id] 2024-11-13T07:15:44,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741826_1002 (size=42) 2024-11-13T07:15:44,193 INFO [master/2998b885bdef:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T07:15:44,194 INFO [master/2998b885bdef:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T07:15:44,195 INFO [master/2998b885bdef:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-13T07:15:44,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39195-0x101330cc6ab0002, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:15:44,208 DEBUG [pool-309-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41365-0x101330cc6ab0001, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:15:44,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:15:44,208 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36543-0x101330cc6ab0003, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:15:44,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741827_1003 (size=196) 2024-11-13T07:15:44,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741827_1003 (size=196) 2024-11-13T07:15:44,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741827_1003 (size=196) 2024-11-13T07:15:44,219 INFO [master/2998b885bdef:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T07:15:44,220 INFO [master/2998b885bdef:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T07:15:44,220 INFO [master/2998b885bdef:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:15:44,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741828_1004 (size=1189) 2024-11-13T07:15:44,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741828_1004 (size=1189) 2024-11-13T07:15:44,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741828_1004 (size=1189) 2024-11-13T07:15:44,473 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-13T07:15:44,477 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T07:15:44,502 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T07:15:44,503 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T07:15:44,635 INFO [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/MasterData/data/master/store 2024-11-13T07:15:44,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741829_1005 (size=34) 2024-11-13T07:15:44,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741829_1005 (size=34) 2024-11-13T07:15:44,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741829_1005 (size=34) 2024-11-13T07:15:44,644 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:15:44,644 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T07:15:44,644 INFO [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T07:15:44,644 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T07:15:44,644 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T07:15:44,644 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T07:15:44,644 INFO [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T07:15:44,644 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731482144644Disabling compacts and flushes for region at 1731482144644Disabling writes for close at 1731482144644Writing region close event to WAL at 1731482144644Closed at 1731482144644 2024-11-13T07:15:44,645 WARN [master/2998b885bdef:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/MasterData/data/master/store/.initializing 2024-11-13T07:15:44,645 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/MasterData/WALs/2998b885bdef,44607,1731482143517 2024-11-13T07:15:44,646 INFO [master/2998b885bdef:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T07:15:44,648 INFO [master/2998b885bdef:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2998b885bdef%2C44607%2C1731482143517, suffix=, logDir=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/MasterData/WALs/2998b885bdef,44607,1731482143517, archiveDir=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/MasterData/oldWALs, maxLogs=10 2024-11-13T07:15:44,661 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/MasterData/WALs/2998b885bdef,44607,1731482143517/2998b885bdef%2C44607%2C1731482143517.1731482144649, exclude list is [], retry=0 2024-11-13T07:15:44,665 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43743,DS-e9be3a87-9173-46cd-97c2-458b82422043,DISK] 2024-11-13T07:15:44,665 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45017,DS-e60c6474-3785-4b26-bb2a-dccf30dc623d,DISK] 2024-11-13T07:15:44,665 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46793,DS-3008da15-b38a-4db4-8f70-4870724e90c0,DISK] 2024-11-13T07:15:44,667 INFO [master/2998b885bdef:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/MasterData/WALs/2998b885bdef,44607,1731482143517/2998b885bdef%2C44607%2C1731482143517.1731482144649 2024-11-13T07:15:44,667 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35119:35119),(127.0.0.1/127.0.0.1:32917:32917),(127.0.0.1/127.0.0.1:41379:41379)] 2024-11-13T07:15:44,667 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:15:44,667 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:15:44,668 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:15:44,668 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:15:44,669 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:15:44,671 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T07:15:44,671 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:15:44,672 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:15:44,672 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:15:44,674 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T07:15:44,674 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:15:44,675 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:15:44,675 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:15:44,676 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T07:15:44,677 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:15:44,677 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:15:44,677 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:15:44,679 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T07:15:44,679 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:15:44,679 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:15:44,679 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:15:44,680 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:15:44,681 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:15:44,682 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:15:44,682 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:15:44,683 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T07:15:44,684 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T07:15:44,686 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:15:44,686 INFO [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70443676, jitterRate=0.04969257116317749}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T07:15:44,687 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731482144668Initializing all the Stores at 1731482144669 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731482144669Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482144669Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482144669Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482144669Cleaning up temporary data from old regions at 1731482144682 (+13 ms)Region opened successfully at 1731482144687 (+5 ms) 2024-11-13T07:15:44,687 INFO [master/2998b885bdef:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T07:15:44,691 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36907dc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2998b885bdef/172.17.0.3:0 2024-11-13T07:15:44,692 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T07:15:44,692 INFO [master/2998b885bdef:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T07:15:44,692 INFO [master/2998b885bdef:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T07:15:44,692 INFO [master/2998b885bdef:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T07:15:44,693 INFO [master/2998b885bdef:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T07:15:44,693 INFO [master/2998b885bdef:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-13T07:15:44,693 INFO [master/2998b885bdef:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=128, operationDelay=10000 2024-11-13T07:15:44,696 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T07:15:44,697 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T07:15:44,717 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T07:15:44,718 INFO [master/2998b885bdef:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T07:15:44,719 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T07:15:44,728 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T07:15:44,728 INFO [master/2998b885bdef:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T07:15:44,729 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T07:15:44,738 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T07:15:44,740 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T07:15:44,770 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T07:15:44,773 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T07:15:44,780 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T07:15:44,791 DEBUG [pool-309-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41365-0x101330cc6ab0001, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T07:15:44,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39195-0x101330cc6ab0002, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T07:15:44,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T07:15:44,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36543-0x101330cc6ab0003, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T07:15:44,791 DEBUG [pool-309-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41365-0x101330cc6ab0001, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:15:44,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36543-0x101330cc6ab0003, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:15:44,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:15:44,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39195-0x101330cc6ab0002, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:15:44,793 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=2998b885bdef,44607,1731482143517, sessionid=0x101330cc6ab0000, setting cluster-up flag (Was=false) 2024-11-13T07:15:44,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36543-0x101330cc6ab0003, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:15:44,812 DEBUG [pool-309-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41365-0x101330cc6ab0001, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:15:44,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39195-0x101330cc6ab0002, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:15:44,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:15:44,844 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T07:15:44,846 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2998b885bdef,44607,1731482143517 2024-11-13T07:15:44,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:15:44,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39195-0x101330cc6ab0002, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:15:44,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36543-0x101330cc6ab0003, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:15:44,865 DEBUG [pool-309-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41365-0x101330cc6ab0001, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:15:44,896 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T07:15:44,897 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2998b885bdef,44607,1731482143517 2024-11-13T07:15:44,898 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T07:15:44,900 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T07:15:44,900 INFO [master/2998b885bdef:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T07:15:44,901 INFO [master/2998b885bdef:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T07:15:44,901 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 2998b885bdef,44607,1731482143517 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T07:15:44,902 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/2998b885bdef:0, corePoolSize=5, maxPoolSize=5 2024-11-13T07:15:44,902 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/2998b885bdef:0, corePoolSize=5, maxPoolSize=5 2024-11-13T07:15:44,902 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/2998b885bdef:0, corePoolSize=5, maxPoolSize=5 2024-11-13T07:15:44,902 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/2998b885bdef:0, corePoolSize=5, maxPoolSize=5 2024-11-13T07:15:44,902 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/2998b885bdef:0, corePoolSize=10, maxPoolSize=10 2024-11-13T07:15:44,902 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:44,902 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/2998b885bdef:0, corePoolSize=2, maxPoolSize=2 2024-11-13T07:15:44,902 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:44,903 INFO [master/2998b885bdef:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731482174903 2024-11-13T07:15:44,903 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T07:15:44,903 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T07:15:44,903 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T07:15:44,904 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T07:15:44,904 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T07:15:44,904 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T07:15:44,904 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:44,904 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T07:15:44,904 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T07:15:44,904 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T07:15:44,904 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T07:15:44,904 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T07:15:44,904 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T07:15:44,905 INFO [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T07:15:44,905 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:15:44,906 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T07:15:44,906 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/2998b885bdef:0:becomeActiveMaster-HFileCleaner.large.0-1731482144905,5,FailOnTimeoutGroup] 2024-11-13T07:15:44,907 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/2998b885bdef:0:becomeActiveMaster-HFileCleaner.small.0-1731482144907,5,FailOnTimeoutGroup] 2024-11-13T07:15:44,907 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:44,907 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T07:15:44,907 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:44,907 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:44,916 INFO [RS:0;2998b885bdef:41365 {}] regionserver.HRegionServer(746): ClusterId : d9509229-0f8c-4541-803c-0e13c5ef5f10 2024-11-13T07:15:44,916 INFO [RS:2;2998b885bdef:36543 {}] regionserver.HRegionServer(746): ClusterId : d9509229-0f8c-4541-803c-0e13c5ef5f10 2024-11-13T07:15:44,916 DEBUG [RS:2;2998b885bdef:36543 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T07:15:44,916 DEBUG [RS:0;2998b885bdef:41365 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T07:15:44,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741831_1007 (size=1321) 2024-11-13T07:15:44,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741831_1007 (size=1321) 2024-11-13T07:15:44,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741831_1007 (size=1321) 2024-11-13T07:15:44,920 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T07:15:44,920 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca 2024-11-13T07:15:44,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741832_1008 (size=32) 2024-11-13T07:15:44,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741832_1008 (size=32) 2024-11-13T07:15:44,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741832_1008 (size=32) 2024-11-13T07:15:44,929 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:15:44,930 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T07:15:44,932 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T07:15:44,932 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:15:44,932 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:15:44,932 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T07:15:44,951 INFO [RS:1;2998b885bdef:39195 {}] regionserver.HRegionServer(746): ClusterId : d9509229-0f8c-4541-803c-0e13c5ef5f10 2024-11-13T07:15:44,951 DEBUG [RS:1;2998b885bdef:39195 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T07:15:44,952 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T07:15:44,952 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:15:44,952 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:15:44,953 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T07:15:44,954 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T07:15:44,954 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:15:44,954 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:15:44,954 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T07:15:44,956 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T07:15:44,956 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:15:44,957 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:15:44,957 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T07:15:44,958 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/hbase/meta/1588230740 2024-11-13T07:15:44,959 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/hbase/meta/1588230740 2024-11-13T07:15:44,960 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T07:15:44,960 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T07:15:44,961 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T07:15:44,963 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T07:15:44,965 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:15:44,966 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62888315, jitterRate=-0.06289108097553253}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T07:15:44,966 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731482144929Initializing all the Stores at 1731482144930 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731482144930Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731482144930Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482144930Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731482144930Cleaning up temporary data from old regions at 1731482144960 (+30 ms)Region opened successfully at 1731482144966 (+6 ms) 2024-11-13T07:15:44,966 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T07:15:44,966 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T07:15:44,966 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T07:15:44,966 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T07:15:44,966 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T07:15:44,966 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T07:15:44,966 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731482144966Disabling compacts and flushes for region at 1731482144966Disabling writes for close at 1731482144966Writing region close event to WAL at 1731482144966Closed at 1731482144966 2024-11-13T07:15:44,967 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T07:15:44,967 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T07:15:44,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T07:15:44,968 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T07:15:44,969 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T07:15:44,998 DEBUG [RS:2;2998b885bdef:36543 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T07:15:44,998 DEBUG [RS:0;2998b885bdef:41365 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T07:15:44,998 DEBUG [RS:0;2998b885bdef:41365 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T07:15:44,998 DEBUG [RS:2;2998b885bdef:36543 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T07:15:45,018 DEBUG [RS:1;2998b885bdef:39195 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T07:15:45,019 DEBUG [RS:1;2998b885bdef:39195 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T07:15:45,029 DEBUG [RS:0;2998b885bdef:41365 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T07:15:45,029 DEBUG [RS:2;2998b885bdef:36543 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T07:15:45,030 DEBUG [RS:0;2998b885bdef:41365 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b3293b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2998b885bdef/172.17.0.3:0 2024-11-13T07:15:45,030 DEBUG [RS:2;2998b885bdef:36543 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61f57af5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2998b885bdef/172.17.0.3:0 2024-11-13T07:15:45,030 DEBUG [RS:1;2998b885bdef:39195 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T07:15:45,030 DEBUG [RS:1;2998b885bdef:39195 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eadfdae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2998b885bdef/172.17.0.3:0 2024-11-13T07:15:45,045 DEBUG [RS:0;2998b885bdef:41365 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;2998b885bdef:41365 2024-11-13T07:15:45,048 INFO [RS:0;2998b885bdef:41365 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T07:15:45,048 INFO [RS:0;2998b885bdef:41365 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T07:15:45,048 DEBUG [RS:0;2998b885bdef:41365 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T07:15:45,049 INFO [RS:0;2998b885bdef:41365 {}] regionserver.HRegionServer(2659): reportForDuty to master=2998b885bdef,44607,1731482143517 with port=41365, startcode=1731482143808 2024-11-13T07:15:45,049 DEBUG [RS:2;2998b885bdef:36543 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;2998b885bdef:36543 2024-11-13T07:15:45,049 DEBUG [RS:0;2998b885bdef:41365 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T07:15:45,049 INFO [RS:2;2998b885bdef:36543 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T07:15:45,049 INFO [RS:2;2998b885bdef:36543 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T07:15:45,051 DEBUG [RS:2;2998b885bdef:36543 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T07:15:45,051 DEBUG [RS:1;2998b885bdef:39195 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;2998b885bdef:39195 2024-11-13T07:15:45,051 INFO [RS:1;2998b885bdef:39195 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T07:15:45,051 INFO [RS:1;2998b885bdef:39195 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T07:15:45,051 DEBUG [RS:1;2998b885bdef:39195 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T07:15:45,052 INFO [RS:2;2998b885bdef:36543 {}] regionserver.HRegionServer(2659): reportForDuty to master=2998b885bdef,44607,1731482143517 with port=36543, startcode=1731482143970 2024-11-13T07:15:45,052 INFO [RS:1;2998b885bdef:39195 {}] regionserver.HRegionServer(2659): reportForDuty to master=2998b885bdef,44607,1731482143517 with port=39195, startcode=1731482143882 2024-11-13T07:15:45,052 INFO [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59699, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T07:15:45,052 DEBUG [RS:2;2998b885bdef:36543 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T07:15:45,052 DEBUG [RS:1;2998b885bdef:39195 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T07:15:45,052 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44607 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2998b885bdef,41365,1731482143808 2024-11-13T07:15:45,052 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44607 {}] master.ServerManager(517): Registering regionserver=2998b885bdef,41365,1731482143808 2024-11-13T07:15:45,054 DEBUG [RS:0;2998b885bdef:41365 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca 2024-11-13T07:15:45,054 DEBUG [RS:0;2998b885bdef:41365 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40989 2024-11-13T07:15:45,054 DEBUG [RS:0;2998b885bdef:41365 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T07:15:45,055 INFO [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36085, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T07:15:45,055 INFO [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41351, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T07:15:45,056 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44607 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2998b885bdef,39195,1731482143882 2024-11-13T07:15:45,056 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44607 {}] master.ServerManager(517): Registering regionserver=2998b885bdef,39195,1731482143882 2024-11-13T07:15:45,058 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44607 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2998b885bdef,36543,1731482143970 2024-11-13T07:15:45,058 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44607 {}] master.ServerManager(517): Registering regionserver=2998b885bdef,36543,1731482143970 2024-11-13T07:15:45,059 DEBUG [RS:1;2998b885bdef:39195 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca 2024-11-13T07:15:45,059 DEBUG [RS:1;2998b885bdef:39195 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40989 2024-11-13T07:15:45,059 DEBUG [RS:1;2998b885bdef:39195 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T07:15:45,060 DEBUG [RS:2;2998b885bdef:36543 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca 2024-11-13T07:15:45,060 DEBUG [RS:2;2998b885bdef:36543 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40989 2024-11-13T07:15:45,060 DEBUG [RS:2;2998b885bdef:36543 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T07:15:45,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T07:15:45,105 DEBUG [RS:0;2998b885bdef:41365 {}] zookeeper.ZKUtil(111): regionserver:41365-0x101330cc6ab0001, quorum=127.0.0.1:63253, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2998b885bdef,41365,1731482143808 2024-11-13T07:15:45,105 WARN [RS:0;2998b885bdef:41365 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T07:15:45,105 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2998b885bdef,41365,1731482143808] 2024-11-13T07:15:45,105 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2998b885bdef,39195,1731482143882] 2024-11-13T07:15:45,105 INFO [RS:0;2998b885bdef:41365 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:15:45,105 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2998b885bdef,36543,1731482143970] 2024-11-13T07:15:45,105 DEBUG [RS:2;2998b885bdef:36543 {}] zookeeper.ZKUtil(111): regionserver:36543-0x101330cc6ab0003, quorum=127.0.0.1:63253, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2998b885bdef,36543,1731482143970 2024-11-13T07:15:45,105 DEBUG [RS:1;2998b885bdef:39195 {}] zookeeper.ZKUtil(111): regionserver:39195-0x101330cc6ab0002, quorum=127.0.0.1:63253, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2998b885bdef,39195,1731482143882 2024-11-13T07:15:45,105 WARN [RS:1;2998b885bdef:39195 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T07:15:45,105 WARN [RS:2;2998b885bdef:36543 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T07:15:45,105 DEBUG [RS:0;2998b885bdef:41365 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,41365,1731482143808 2024-11-13T07:15:45,105 INFO [RS:1;2998b885bdef:39195 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:15:45,105 INFO [RS:2;2998b885bdef:36543 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:15:45,105 DEBUG [RS:1;2998b885bdef:39195 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,39195,1731482143882 2024-11-13T07:15:45,105 DEBUG [RS:2;2998b885bdef:36543 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,36543,1731482143970 2024-11-13T07:15:45,111 INFO [RS:0;2998b885bdef:41365 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T07:15:45,111 INFO [RS:2;2998b885bdef:36543 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T07:15:45,113 INFO [RS:1;2998b885bdef:39195 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T07:15:45,114 INFO [RS:0;2998b885bdef:41365 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T07:15:45,115 INFO [RS:2;2998b885bdef:36543 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T07:15:45,116 INFO [RS:1;2998b885bdef:39195 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T07:15:45,116 INFO [RS:0;2998b885bdef:41365 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T07:15:45,116 INFO [RS:0;2998b885bdef:41365 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:45,116 INFO [RS:2;2998b885bdef:36543 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T07:15:45,116 INFO [RS:2;2998b885bdef:36543 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:45,116 INFO [RS:1;2998b885bdef:39195 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T07:15:45,116 INFO [RS:1;2998b885bdef:39195 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:45,119 INFO [RS:0;2998b885bdef:41365 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T07:15:45,119 INFO [RS:2;2998b885bdef:36543 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T07:15:45,119 INFO [RS:1;2998b885bdef:39195 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T07:15:45,119 WARN [2998b885bdef:44607 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T07:15:45,120 INFO [RS:1;2998b885bdef:39195 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T07:15:45,120 INFO [RS:2;2998b885bdef:36543 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T07:15:45,120 INFO [RS:0;2998b885bdef:41365 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T07:15:45,120 INFO [RS:1;2998b885bdef:39195 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:45,120 INFO [RS:0;2998b885bdef:41365 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:45,120 INFO [RS:2;2998b885bdef:36543 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:45,120 DEBUG [RS:2;2998b885bdef:36543 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,120 DEBUG [RS:0;2998b885bdef:41365 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,120 DEBUG [RS:1;2998b885bdef:39195 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,120 DEBUG [RS:2;2998b885bdef:36543 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,120 DEBUG [RS:0;2998b885bdef:41365 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,120 DEBUG [RS:1;2998b885bdef:39195 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,120 DEBUG [RS:2;2998b885bdef:36543 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,120 DEBUG [RS:0;2998b885bdef:41365 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,120 DEBUG [RS:1;2998b885bdef:39195 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,120 DEBUG [RS:2;2998b885bdef:36543 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,120 DEBUG [RS:0;2998b885bdef:41365 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,120 DEBUG [RS:1;2998b885bdef:39195 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,121 DEBUG [RS:1;2998b885bdef:39195 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,121 DEBUG [RS:2;2998b885bdef:36543 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,121 DEBUG [RS:0;2998b885bdef:41365 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,121 DEBUG [RS:1;2998b885bdef:39195 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0, corePoolSize=2, maxPoolSize=2 2024-11-13T07:15:45,121 DEBUG [RS:2;2998b885bdef:36543 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0, corePoolSize=2, maxPoolSize=2 2024-11-13T07:15:45,121 DEBUG [RS:0;2998b885bdef:41365 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0, corePoolSize=2, maxPoolSize=2 2024-11-13T07:15:45,121 DEBUG [RS:1;2998b885bdef:39195 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,121 DEBUG [RS:1;2998b885bdef:39195 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,121 DEBUG [RS:2;2998b885bdef:36543 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,121 DEBUG [RS:0;2998b885bdef:41365 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,121 DEBUG [RS:1;2998b885bdef:39195 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,121 DEBUG [RS:2;2998b885bdef:36543 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,121 DEBUG [RS:0;2998b885bdef:41365 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,121 DEBUG [RS:1;2998b885bdef:39195 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,121 DEBUG [RS:2;2998b885bdef:36543 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,121 DEBUG [RS:0;2998b885bdef:41365 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,121 DEBUG [RS:1;2998b885bdef:39195 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,121 DEBUG [RS:2;2998b885bdef:36543 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,121 DEBUG [RS:0;2998b885bdef:41365 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,121 DEBUG [RS:1;2998b885bdef:39195 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,121 DEBUG [RS:2;2998b885bdef:36543 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,121 DEBUG [RS:0;2998b885bdef:41365 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,121 DEBUG [RS:1;2998b885bdef:39195 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0, corePoolSize=3, maxPoolSize=3 2024-11-13T07:15:45,121 DEBUG [RS:2;2998b885bdef:36543 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,121 DEBUG [RS:0;2998b885bdef:41365 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2998b885bdef:0, corePoolSize=1, maxPoolSize=1 2024-11-13T07:15:45,121 DEBUG [RS:1;2998b885bdef:39195 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2998b885bdef:0, corePoolSize=3, maxPoolSize=3 2024-11-13T07:15:45,121 DEBUG [RS:0;2998b885bdef:41365 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0, corePoolSize=3, maxPoolSize=3 2024-11-13T07:15:45,121 DEBUG [RS:2;2998b885bdef:36543 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0, corePoolSize=3, maxPoolSize=3 2024-11-13T07:15:45,121 DEBUG [RS:2;2998b885bdef:36543 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2998b885bdef:0, corePoolSize=3, maxPoolSize=3 2024-11-13T07:15:45,121 DEBUG [RS:0;2998b885bdef:41365 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2998b885bdef:0, corePoolSize=3, maxPoolSize=3 2024-11-13T07:15:45,124 INFO [RS:1;2998b885bdef:39195 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:45,124 INFO [RS:1;2998b885bdef:39195 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:45,124 INFO [RS:1;2998b885bdef:39195 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:45,124 INFO [RS:1;2998b885bdef:39195 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:45,124 INFO [RS:1;2998b885bdef:39195 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:45,124 INFO [RS:1;2998b885bdef:39195 {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,39195,1731482143882-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T07:15:45,124 INFO [RS:2;2998b885bdef:36543 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:45,125 INFO [RS:2;2998b885bdef:36543 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:45,125 INFO [RS:0;2998b885bdef:41365 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:45,125 INFO [RS:2;2998b885bdef:36543 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:45,125 INFO [RS:0;2998b885bdef:41365 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:45,125 INFO [RS:2;2998b885bdef:36543 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:45,125 INFO [RS:0;2998b885bdef:41365 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:45,125 INFO [RS:2;2998b885bdef:36543 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:45,125 INFO [RS:0;2998b885bdef:41365 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:45,125 INFO [RS:2;2998b885bdef:36543 {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,36543,1731482143970-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T07:15:45,125 INFO [RS:0;2998b885bdef:41365 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:45,125 INFO [RS:0;2998b885bdef:41365 {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,41365,1731482143808-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T07:15:45,142 INFO [RS:2;2998b885bdef:36543 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T07:15:45,142 INFO [RS:1;2998b885bdef:39195 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T07:15:45,142 INFO [RS:1;2998b885bdef:39195 {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,39195,1731482143882-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:45,142 INFO [RS:2;2998b885bdef:36543 {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,36543,1731482143970-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:45,142 INFO [RS:2;2998b885bdef:36543 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:45,142 INFO [RS:1;2998b885bdef:39195 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:45,142 INFO [RS:2;2998b885bdef:36543 {}] regionserver.Replication(171): 2998b885bdef,36543,1731482143970 started 2024-11-13T07:15:45,142 INFO [RS:1;2998b885bdef:39195 {}] regionserver.Replication(171): 2998b885bdef,39195,1731482143882 started 2024-11-13T07:15:45,142 INFO [RS:0;2998b885bdef:41365 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T07:15:45,142 INFO [RS:0;2998b885bdef:41365 {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,41365,1731482143808-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:45,142 INFO [RS:0;2998b885bdef:41365 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:45,142 INFO [RS:0;2998b885bdef:41365 {}] regionserver.Replication(171): 2998b885bdef,41365,1731482143808 started 2024-11-13T07:15:45,155 INFO [RS:2;2998b885bdef:36543 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:45,155 INFO [RS:1;2998b885bdef:39195 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:45,155 INFO [RS:1;2998b885bdef:39195 {}] regionserver.HRegionServer(1482): Serving as 2998b885bdef,39195,1731482143882, RpcServer on 2998b885bdef/172.17.0.3:39195, sessionid=0x101330cc6ab0002 2024-11-13T07:15:45,155 INFO [RS:2;2998b885bdef:36543 {}] regionserver.HRegionServer(1482): Serving as 2998b885bdef,36543,1731482143970, RpcServer on 2998b885bdef/172.17.0.3:36543, sessionid=0x101330cc6ab0003 2024-11-13T07:15:45,155 DEBUG [RS:2;2998b885bdef:36543 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T07:15:45,155 DEBUG [RS:1;2998b885bdef:39195 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T07:15:45,155 DEBUG [RS:1;2998b885bdef:39195 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2998b885bdef,39195,1731482143882 2024-11-13T07:15:45,155 DEBUG [RS:2;2998b885bdef:36543 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2998b885bdef,36543,1731482143970 2024-11-13T07:15:45,155 DEBUG [RS:1;2998b885bdef:39195 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2998b885bdef,39195,1731482143882' 2024-11-13T07:15:45,155 DEBUG [RS:2;2998b885bdef:36543 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2998b885bdef,36543,1731482143970' 2024-11-13T07:15:45,155 DEBUG [RS:1;2998b885bdef:39195 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T07:15:45,155 DEBUG [RS:2;2998b885bdef:36543 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T07:15:45,156 DEBUG [RS:2;2998b885bdef:36543 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T07:15:45,156 DEBUG [RS:1;2998b885bdef:39195 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T07:15:45,156 DEBUG [RS:2;2998b885bdef:36543 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T07:15:45,156 DEBUG [RS:1;2998b885bdef:39195 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T07:15:45,156 DEBUG [RS:2;2998b885bdef:36543 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T07:15:45,156 DEBUG [RS:1;2998b885bdef:39195 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T07:15:45,156 DEBUG [RS:2;2998b885bdef:36543 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2998b885bdef,36543,1731482143970 2024-11-13T07:15:45,156 DEBUG [RS:1;2998b885bdef:39195 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2998b885bdef,39195,1731482143882 2024-11-13T07:15:45,156 DEBUG [RS:2;2998b885bdef:36543 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2998b885bdef,36543,1731482143970' 2024-11-13T07:15:45,156 DEBUG [RS:1;2998b885bdef:39195 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2998b885bdef,39195,1731482143882' 2024-11-13T07:15:45,156 DEBUG [RS:2;2998b885bdef:36543 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T07:15:45,156 DEBUG [RS:1;2998b885bdef:39195 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T07:15:45,157 DEBUG [RS:1;2998b885bdef:39195 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T07:15:45,157 DEBUG [RS:2;2998b885bdef:36543 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T07:15:45,157 DEBUG [RS:1;2998b885bdef:39195 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T07:15:45,157 INFO [RS:1;2998b885bdef:39195 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T07:15:45,157 INFO [RS:1;2998b885bdef:39195 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T07:15:45,157 DEBUG [RS:2;2998b885bdef:36543 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T07:15:45,157 INFO [RS:2;2998b885bdef:36543 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T07:15:45,157 INFO [RS:2;2998b885bdef:36543 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T07:15:45,157 INFO [RS:0;2998b885bdef:41365 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:45,158 INFO [RS:0;2998b885bdef:41365 {}] regionserver.HRegionServer(1482): Serving as 2998b885bdef,41365,1731482143808, RpcServer on 2998b885bdef/172.17.0.3:41365, sessionid=0x101330cc6ab0001 2024-11-13T07:15:45,158 DEBUG [RS:0;2998b885bdef:41365 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T07:15:45,158 DEBUG [RS:0;2998b885bdef:41365 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2998b885bdef,41365,1731482143808 2024-11-13T07:15:45,158 DEBUG [RS:0;2998b885bdef:41365 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2998b885bdef,41365,1731482143808' 2024-11-13T07:15:45,158 DEBUG [RS:0;2998b885bdef:41365 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T07:15:45,158 DEBUG [RS:0;2998b885bdef:41365 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T07:15:45,158 DEBUG [RS:0;2998b885bdef:41365 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T07:15:45,158 DEBUG [RS:0;2998b885bdef:41365 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T07:15:45,158 DEBUG [RS:0;2998b885bdef:41365 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2998b885bdef,41365,1731482143808 2024-11-13T07:15:45,158 DEBUG [RS:0;2998b885bdef:41365 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2998b885bdef,41365,1731482143808' 2024-11-13T07:15:45,158 DEBUG [RS:0;2998b885bdef:41365 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T07:15:45,159 DEBUG [RS:0;2998b885bdef:41365 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T07:15:45,159 DEBUG [RS:0;2998b885bdef:41365 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T07:15:45,160 INFO [RS:0;2998b885bdef:41365 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T07:15:45,160 INFO [RS:0;2998b885bdef:41365 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T07:15:45,258 INFO [RS:1;2998b885bdef:39195 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T07:15:45,258 INFO [RS:2;2998b885bdef:36543 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T07:15:45,260 INFO [RS:0;2998b885bdef:41365 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T07:15:45,262 INFO [RS:1;2998b885bdef:39195 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2998b885bdef%2C39195%2C1731482143882, suffix=, logDir=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,39195,1731482143882, archiveDir=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/oldWALs, maxLogs=32 2024-11-13T07:15:45,262 INFO [RS:2;2998b885bdef:36543 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2998b885bdef%2C36543%2C1731482143970, suffix=, logDir=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,36543,1731482143970, archiveDir=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/oldWALs, maxLogs=32 2024-11-13T07:15:45,264 INFO [RS:0;2998b885bdef:41365 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2998b885bdef%2C41365%2C1731482143808, suffix=, logDir=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,41365,1731482143808, archiveDir=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/oldWALs, maxLogs=32 2024-11-13T07:15:45,281 DEBUG [RS:1;2998b885bdef:39195 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,39195,1731482143882/2998b885bdef%2C39195%2C1731482143882.1731482145268, exclude list is [], retry=0 2024-11-13T07:15:45,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45017,DS-e60c6474-3785-4b26-bb2a-dccf30dc623d,DISK] 2024-11-13T07:15:45,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46793,DS-3008da15-b38a-4db4-8f70-4870724e90c0,DISK] 2024-11-13T07:15:45,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43743,DS-e9be3a87-9173-46cd-97c2-458b82422043,DISK] 2024-11-13T07:15:45,287 INFO [RS:1;2998b885bdef:39195 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,39195,1731482143882/2998b885bdef%2C39195%2C1731482143882.1731482145268 2024-11-13T07:15:45,287 DEBUG [RS:2;2998b885bdef:36543 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,36543,1731482143970/2998b885bdef%2C36543%2C1731482143970.1731482145268, exclude list is [], retry=0 2024-11-13T07:15:45,287 DEBUG [RS:1;2998b885bdef:39195 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41379:41379),(127.0.0.1/127.0.0.1:32917:32917),(127.0.0.1/127.0.0.1:35119:35119)] 2024-11-13T07:15:45,287 DEBUG [RS:0;2998b885bdef:41365 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,41365,1731482143808/2998b885bdef%2C41365%2C1731482143808.1731482145268, exclude list is [], retry=0 2024-11-13T07:15:45,290 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46793,DS-3008da15-b38a-4db4-8f70-4870724e90c0,DISK] 2024-11-13T07:15:45,290 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46793,DS-3008da15-b38a-4db4-8f70-4870724e90c0,DISK] 2024-11-13T07:15:45,290 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45017,DS-e60c6474-3785-4b26-bb2a-dccf30dc623d,DISK] 2024-11-13T07:15:45,290 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45017,DS-e60c6474-3785-4b26-bb2a-dccf30dc623d,DISK] 2024-11-13T07:15:45,290 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43743,DS-e9be3a87-9173-46cd-97c2-458b82422043,DISK] 2024-11-13T07:15:45,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43743,DS-e9be3a87-9173-46cd-97c2-458b82422043,DISK] 2024-11-13T07:15:45,298 INFO [RS:2;2998b885bdef:36543 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,36543,1731482143970/2998b885bdef%2C36543%2C1731482143970.1731482145268 2024-11-13T07:15:45,298 DEBUG [RS:2;2998b885bdef:36543 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41379:41379),(127.0.0.1/127.0.0.1:32917:32917),(127.0.0.1/127.0.0.1:35119:35119)] 2024-11-13T07:15:45,301 INFO [RS:0;2998b885bdef:41365 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,41365,1731482143808/2998b885bdef%2C41365%2C1731482143808.1731482145268 2024-11-13T07:15:45,301 DEBUG [RS:0;2998b885bdef:41365 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32917:32917),(127.0.0.1/127.0.0.1:41379:41379),(127.0.0.1/127.0.0.1:35119:35119)] 2024-11-13T07:15:45,370 DEBUG [2998b885bdef:44607 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-13T07:15:45,371 DEBUG [2998b885bdef:44607 {}] balancer.BalancerClusterState(204): Hosts are {2998b885bdef=0} racks are {/default-rack=0} 2024-11-13T07:15:45,375 DEBUG [2998b885bdef:44607 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T07:15:45,375 DEBUG [2998b885bdef:44607 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T07:15:45,375 DEBUG [2998b885bdef:44607 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-13T07:15:45,375 DEBUG [2998b885bdef:44607 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T07:15:45,375 DEBUG [2998b885bdef:44607 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T07:15:45,375 DEBUG [2998b885bdef:44607 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T07:15:45,375 INFO [2998b885bdef:44607 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T07:15:45,375 INFO [2998b885bdef:44607 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T07:15:45,376 INFO [2998b885bdef:44607 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T07:15:45,376 DEBUG [2998b885bdef:44607 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T07:15:45,376 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2998b885bdef,36543,1731482143970 2024-11-13T07:15:45,377 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2998b885bdef,36543,1731482143970, state=OPENING 2024-11-13T07:15:45,397 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T07:15:45,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39195-0x101330cc6ab0002, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:15:45,408 DEBUG [pool-309-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41365-0x101330cc6ab0001, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:15:45,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36543-0x101330cc6ab0003, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:15:45,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:15:45,409 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:15:45,409 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:15:45,409 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T07:15:45,410 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:15:45,410 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:15:45,410 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2998b885bdef,36543,1731482143970}] 2024-11-13T07:15:55,414 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T07:15:55,416 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46967, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T07:15:55,431 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T07:15:55,431 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T07:15:55,432 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T07:15:55,434 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2998b885bdef%2C36543%2C1731482143970.meta, suffix=.meta, logDir=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,36543,1731482143970, archiveDir=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/oldWALs, maxLogs=32 2024-11-13T07:15:55,456 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,36543,1731482143970/2998b885bdef%2C36543%2C1731482143970.meta.1731482155435.meta, exclude list is [], retry=0 2024-11-13T07:15:55,459 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46793,DS-3008da15-b38a-4db4-8f70-4870724e90c0,DISK] 2024-11-13T07:15:55,460 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43743,DS-e9be3a87-9173-46cd-97c2-458b82422043,DISK] 2024-11-13T07:15:55,463 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45017,DS-e60c6474-3785-4b26-bb2a-dccf30dc623d,DISK] 2024-11-13T07:15:55,468 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,36543,1731482143970/2998b885bdef%2C36543%2C1731482143970.meta.1731482155435.meta 2024-11-13T07:15:55,470 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41379:41379),(127.0.0.1/127.0.0.1:35119:35119),(127.0.0.1/127.0.0.1:32917:32917)] 2024-11-13T07:15:55,470 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T07:15:55,471 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T07:15:55,471 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T07:15:55,471 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T07:15:55,471 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T07:15:55,471 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:15:55,472 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T07:15:55,472 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T07:15:55,480 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T07:15:55,485 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T07:15:55,485 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:15:55,489 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:15:55,489 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T07:15:55,490 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T07:15:55,490 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:15:55,491 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:15:55,491 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T07:15:55,494 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T07:15:55,494 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:15:55,495 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:15:55,495 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T07:15:55,496 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T07:15:55,496 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:15:55,502 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T07:15:55,503 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T07:15:55,504 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/hbase/meta/1588230740 2024-11-13T07:15:55,506 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/hbase/meta/1588230740 2024-11-13T07:15:55,508 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T07:15:55,508 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T07:15:55,509 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T07:15:55,512 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T07:15:55,515 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75030760, jitterRate=0.11804544925689697}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T07:15:55,515 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T07:15:55,515 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731482155472Writing region info on filesystem at 1731482155472Initializing all the Stores at 1731482155473 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731482155473Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731482155480 (+7 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482155480Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731482155480Cleaning up temporary data from old regions at 1731482155508 (+28 ms)Running coprocessor post-open hooks at 1731482155515 (+7 ms)Region opened successfully at 1731482155515 2024-11-13T07:15:55,517 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731482155414 2024-11-13T07:15:55,520 DEBUG [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T07:15:55,520 INFO [RS_OPEN_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T07:15:55,520 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=2998b885bdef,36543,1731482143970 2024-11-13T07:15:55,522 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2998b885bdef,36543,1731482143970, state=OPEN 2024-11-13T07:15:55,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36543-0x101330cc6ab0003, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T07:15:55,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39195-0x101330cc6ab0002, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T07:15:55,544 DEBUG [pool-309-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41365-0x101330cc6ab0001, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T07:15:55,544 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:15:55,544 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:15:55,544 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:15:55,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T07:15:55,545 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T07:15:55,548 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2998b885bdef,36543,1731482143970 2024-11-13T07:15:55,552 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T07:15:55,552 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2998b885bdef,36543,1731482143970 in 10.1380 sec 2024-11-13T07:15:55,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T07:15:55,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 10.5850 sec 2024-11-13T07:15:55,560 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T07:15:55,560 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T07:15:55,589 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T07:15:55,589 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2998b885bdef,36543,1731482143970, seqNum=-1] 2024-11-13T07:15:55,590 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T07:15:55,592 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60503, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T07:15:55,600 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 10.6990 sec 2024-11-13T07:15:55,601 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731482155601, completionTime=-1 2024-11-13T07:15:55,601 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-13T07:15:55,601 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T07:15:55,603 INFO [master/2998b885bdef:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-13T07:15:55,603 INFO [master/2998b885bdef:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731482215603 2024-11-13T07:15:55,603 INFO [master/2998b885bdef:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731482275603 2024-11-13T07:15:55,603 INFO [master/2998b885bdef:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 2 msec 2024-11-13T07:15:55,604 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-13T07:15:55,608 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,44607,1731482143517-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:55,608 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,44607,1731482143517-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:55,609 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,44607,1731482143517-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:55,609 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-2998b885bdef:44607, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:55,609 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:55,609 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:55,613 DEBUG [master/2998b885bdef:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T07:15:55,615 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 11.455sec 2024-11-13T07:15:55,616 INFO [master/2998b885bdef:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T07:15:55,616 INFO [master/2998b885bdef:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T07:15:55,616 INFO [master/2998b885bdef:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T07:15:55,616 INFO [master/2998b885bdef:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T07:15:55,616 INFO [master/2998b885bdef:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T07:15:55,616 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,44607,1731482143517-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T07:15:55,616 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,44607,1731482143517-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T07:15:55,619 DEBUG [master/2998b885bdef:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T07:15:55,619 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T07:15:55,619 INFO [master/2998b885bdef:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2998b885bdef,44607,1731482143517-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T07:15:55,627 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60545ae0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T07:15:55,627 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2998b885bdef,44607,-1 for getting cluster id 2024-11-13T07:15:55,628 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T07:15:55,629 DEBUG [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9509229-0f8c-4541-803c-0e13c5ef5f10' 2024-11-13T07:15:55,630 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T07:15:55,630 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9509229-0f8c-4541-803c-0e13c5ef5f10" 2024-11-13T07:15:55,631 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4afea623, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T07:15:55,631 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2998b885bdef,44607,-1] 2024-11-13T07:15:55,631 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T07:15:55,632 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:15:55,633 INFO [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T07:15:55,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68722710, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T07:15:55,635 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T07:15:55,636 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2998b885bdef,36543,1731482143970, seqNum=-1] 2024-11-13T07:15:55,636 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T07:15:55,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41298, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T07:15:55,648 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=2998b885bdef,44607,1731482143517 2024-11-13T07:15:55,648 DEBUG [Time-limited test {}] snapshot.SnapshotDescriptionUtils(316): Creation time not specified, setting to:1731482155648 (current time:1731482155648). 2024-11-13T07:15:55,648 DEBUG [Time-limited test {}] snapshot.SnapshotDescriptionUtils(330): Snapshot current TTL value: 0 resetting it to default value: 0 2024-11-13T07:15:55,648 DEBUG [Time-limited test {}] snapshot.SnapshotDescriptionUtils(339): Snapshot SnapshotProcedureTest VERSION not specified, setting to 2 2024-11-13T07:15:55,649 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5635039d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T07:15:55,649 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2998b885bdef,44607,-1 for getting cluster id 2024-11-13T07:15:55,649 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T07:15:55,650 DEBUG [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd9509229-0f8c-4541-803c-0e13c5ef5f10' 2024-11-13T07:15:55,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T07:15:55,650 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d9509229-0f8c-4541-803c-0e13c5ef5f10" 2024-11-13T07:15:55,651 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4101c878, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T07:15:55,651 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2998b885bdef,44607,-1] 2024-11-13T07:15:55,651 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T07:15:55,651 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:15:55,653 INFO [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T07:15:55,654 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5db73089, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T07:15:55,655 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T07:15:55,657 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2998b885bdef,36543,1731482143970, seqNum=-1] 2024-11-13T07:15:55,657 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T07:15:55,658 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T07:15:55,659 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T07:15:55,660 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.isSecurityAvailable(SnapshotDescriptionUtils.java:481) at org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils.validate(SnapshotDescriptionUtils.java:353) at org.apache.hadoop.hbase.master.procedure.TestSnapshotProcedure.setup(TestSnapshotProcedure.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunBefores.invokeMethod(RunBefores.java:33) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T07:15:55,660 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:15:55,660 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:15:55,660 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T07:15:55,660 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T07:15:55,662 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 2998b885bdef,44607,1731482143517 2024-11-13T07:15:55,662 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@36dc34c9 2024-11-13T07:15:55,662 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T07:15:55,665 INFO [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53596, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T07:15:55,667 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'SPTestTable', {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T07:15:55,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=SPTestTable 2024-11-13T07:15:55,670 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=SPTestTable execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T07:15:55,670 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:15:55,670 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "SPTestTable" procId is: 4 2024-11-13T07:15:55,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T07:15:55,672 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=SPTestTable execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T07:15:55,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741837_1013 (size=361) 2024-11-13T07:15:55,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741837_1013 (size=361) 2024-11-13T07:15:55,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741837_1013 (size=361) 2024-11-13T07:15:55,697 INFO [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2e4b72fe1d04f499cf9c51ce52c79100, NAME => 'SPTestTable,,1731482155666.2e4b72fe1d04f499cf9c51ce52c79100.', STARTKEY => '', ENDKEY => '19999999'}, tableDescriptor='SPTestTable', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca 2024-11-13T07:15:55,698 INFO [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1f49ca9e4320a113a985ca82c73f5c2a, NAME => 'SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a.', STARTKEY => '19999999', ENDKEY => '33333332'}, tableDescriptor='SPTestTable', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca 2024-11-13T07:15:55,700 INFO [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 045378476123254a5c2616896d4edfcf, NAME => 'SPTestTable,33333332,1731482155666.045378476123254a5c2616896d4edfcf.', STARTKEY => '33333332', ENDKEY => '4ccccccb'}, tableDescriptor='SPTestTable', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca 2024-11-13T07:15:55,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741839_1015 (size=62) 2024-11-13T07:15:55,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741838_1014 (size=54) 2024-11-13T07:15:55,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741839_1015 (size=62) 2024-11-13T07:15:55,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741839_1015 (size=62) 2024-11-13T07:15:55,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741838_1014 (size=54) 2024-11-13T07:15:55,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741838_1014 (size=54) 2024-11-13T07:15:55,717 DEBUG [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(898): Instantiated SPTestTable,,1731482155666.2e4b72fe1d04f499cf9c51ce52c79100.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:15:55,717 DEBUG [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(1722): Closing 2e4b72fe1d04f499cf9c51ce52c79100, disabling compactions & flushes 2024-11-13T07:15:55,717 INFO [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(1755): Closing region SPTestTable,,1731482155666.2e4b72fe1d04f499cf9c51ce52c79100. 2024-11-13T07:15:55,717 DEBUG [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,,1731482155666.2e4b72fe1d04f499cf9c51ce52c79100. 2024-11-13T07:15:55,717 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(898): Instantiated SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:15:55,717 DEBUG [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,,1731482155666.2e4b72fe1d04f499cf9c51ce52c79100. after waiting 0 ms 2024-11-13T07:15:55,717 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1722): Closing 1f49ca9e4320a113a985ca82c73f5c2a, disabling compactions & flushes 2024-11-13T07:15:55,717 DEBUG [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,,1731482155666.2e4b72fe1d04f499cf9c51ce52c79100. 2024-11-13T07:15:55,717 INFO [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1755): Closing region SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a. 2024-11-13T07:15:55,717 INFO [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(1973): Closed SPTestTable,,1731482155666.2e4b72fe1d04f499cf9c51ce52c79100. 2024-11-13T07:15:55,717 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a. 2024-11-13T07:15:55,717 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a. after waiting 0 ms 2024-11-13T07:15:55,718 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a. 2024-11-13T07:15:55,718 DEBUG [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2e4b72fe1d04f499cf9c51ce52c79100: Waiting for close lock at 1731482155717Disabling compacts and flushes for region at 1731482155717Disabling writes for close at 1731482155717Writing region close event to WAL at 1731482155717Closed at 1731482155717 2024-11-13T07:15:55,718 INFO [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1973): Closed SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a. 2024-11-13T07:15:55,718 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1676): Region close journal for 1f49ca9e4320a113a985ca82c73f5c2a: Waiting for close lock at 1731482155717Disabling compacts and flushes for region at 1731482155717Disabling writes for close at 1731482155718 (+1 ms)Writing region close event to WAL at 1731482155718Closed at 1731482155718 2024-11-13T07:15:55,718 INFO [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ff8b864839731879b51af65fe2a46ac9, NAME => 'SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9.', STARTKEY => '4ccccccb', ENDKEY => '66666664'}, tableDescriptor='SPTestTable', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca 2024-11-13T07:15:55,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741840_1016 (size=62) 2024-11-13T07:15:55,719 INFO [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1357138ab69f6578d72ce8d295f98b00, NAME => 'SPTestTable,66666664,1731482155666.1357138ab69f6578d72ce8d295f98b00.', STARTKEY => '66666664', ENDKEY => '7ffffffd'}, tableDescriptor='SPTestTable', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca 2024-11-13T07:15:55,721 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(898): Instantiated SPTestTable,33333332,1731482155666.045378476123254a5c2616896d4edfcf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:15:55,722 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1722): Closing 045378476123254a5c2616896d4edfcf, disabling compactions & flushes 2024-11-13T07:15:55,722 INFO [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1755): Closing region SPTestTable,33333332,1731482155666.045378476123254a5c2616896d4edfcf. 2024-11-13T07:15:55,722 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,33333332,1731482155666.045378476123254a5c2616896d4edfcf. 2024-11-13T07:15:55,722 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,33333332,1731482155666.045378476123254a5c2616896d4edfcf. after waiting 0 ms 2024-11-13T07:15:55,722 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,33333332,1731482155666.045378476123254a5c2616896d4edfcf. 2024-11-13T07:15:55,722 INFO [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1973): Closed SPTestTable,33333332,1731482155666.045378476123254a5c2616896d4edfcf. 2024-11-13T07:15:55,722 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1676): Region close journal for 045378476123254a5c2616896d4edfcf: Waiting for close lock at 1731482155722Disabling compacts and flushes for region at 1731482155722Disabling writes for close at 1731482155722Writing region close event to WAL at 1731482155722Closed at 1731482155722 2024-11-13T07:15:55,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741840_1016 (size=62) 2024-11-13T07:15:55,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741840_1016 (size=62) 2024-11-13T07:15:55,723 INFO [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 9e8af980aa626886ac3cb035f55a2e4c, NAME => 'SPTestTable,7ffffffd,1731482155666.9e8af980aa626886ac3cb035f55a2e4c.', STARTKEY => '7ffffffd', ENDKEY => '99999996'}, tableDescriptor='SPTestTable', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca 2024-11-13T07:15:55,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741841_1017 (size=62) 2024-11-13T07:15:55,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741841_1017 (size=62) 2024-11-13T07:15:55,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741841_1017 (size=62) 2024-11-13T07:15:55,735 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(898): Instantiated SPTestTable,66666664,1731482155666.1357138ab69f6578d72ce8d295f98b00.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:15:55,735 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1722): Closing 1357138ab69f6578d72ce8d295f98b00, disabling compactions & flushes 2024-11-13T07:15:55,735 INFO [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1755): Closing region SPTestTable,66666664,1731482155666.1357138ab69f6578d72ce8d295f98b00. 2024-11-13T07:15:55,736 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,66666664,1731482155666.1357138ab69f6578d72ce8d295f98b00. 2024-11-13T07:15:55,736 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,66666664,1731482155666.1357138ab69f6578d72ce8d295f98b00. after waiting 0 ms 2024-11-13T07:15:55,736 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,66666664,1731482155666.1357138ab69f6578d72ce8d295f98b00. 2024-11-13T07:15:55,736 INFO [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1973): Closed SPTestTable,66666664,1731482155666.1357138ab69f6578d72ce8d295f98b00. 2024-11-13T07:15:55,736 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1676): Region close journal for 1357138ab69f6578d72ce8d295f98b00: Waiting for close lock at 1731482155735Disabling compacts and flushes for region at 1731482155735Disabling writes for close at 1731482155736 (+1 ms)Writing region close event to WAL at 1731482155736Closed at 1731482155736 2024-11-13T07:15:55,737 INFO [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => dc0d3e1b8b1a5a44332ecc1f0118002b, NAME => 'SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b.', STARTKEY => '99999996', ENDKEY => 'b333332f'}, tableDescriptor='SPTestTable', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca 2024-11-13T07:15:55,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741842_1018 (size=62) 2024-11-13T07:15:55,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741842_1018 (size=62) 2024-11-13T07:15:55,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741842_1018 (size=62) 2024-11-13T07:15:55,746 DEBUG [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(898): Instantiated SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:15:55,746 DEBUG [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(1722): Closing ff8b864839731879b51af65fe2a46ac9, disabling compactions & flushes 2024-11-13T07:15:55,746 INFO [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(1755): Closing region SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9. 2024-11-13T07:15:55,747 DEBUG [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9. 2024-11-13T07:15:55,747 DEBUG [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9. after waiting 0 ms 2024-11-13T07:15:55,747 DEBUG [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9. 2024-11-13T07:15:55,747 INFO [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(1973): Closed SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9. 2024-11-13T07:15:55,747 DEBUG [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(1676): Region close journal for ff8b864839731879b51af65fe2a46ac9: Waiting for close lock at 1731482155746Disabling compacts and flushes for region at 1731482155746Disabling writes for close at 1731482155747 (+1 ms)Writing region close event to WAL at 1731482155747Closed at 1731482155747 2024-11-13T07:15:55,748 INFO [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 56b34ea82217cf90ce43d525dd7e1a9d, NAME => 'SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d.', STARTKEY => 'b333332f', ENDKEY => 'ccccccc8'}, tableDescriptor='SPTestTable', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca 2024-11-13T07:15:55,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741843_1019 (size=62) 2024-11-13T07:15:55,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741843_1019 (size=62) 2024-11-13T07:15:55,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741843_1019 (size=62) 2024-11-13T07:15:55,755 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(898): Instantiated SPTestTable,7ffffffd,1731482155666.9e8af980aa626886ac3cb035f55a2e4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:15:55,755 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1722): Closing 9e8af980aa626886ac3cb035f55a2e4c, disabling compactions & flushes 2024-11-13T07:15:55,755 INFO [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1755): Closing region SPTestTable,7ffffffd,1731482155666.9e8af980aa626886ac3cb035f55a2e4c. 2024-11-13T07:15:55,755 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,7ffffffd,1731482155666.9e8af980aa626886ac3cb035f55a2e4c. 2024-11-13T07:15:55,755 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,7ffffffd,1731482155666.9e8af980aa626886ac3cb035f55a2e4c. after waiting 0 ms 2024-11-13T07:15:55,755 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,7ffffffd,1731482155666.9e8af980aa626886ac3cb035f55a2e4c. 2024-11-13T07:15:55,755 INFO [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1973): Closed SPTestTable,7ffffffd,1731482155666.9e8af980aa626886ac3cb035f55a2e4c. 2024-11-13T07:15:55,755 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1676): Region close journal for 9e8af980aa626886ac3cb035f55a2e4c: Waiting for close lock at 1731482155755Disabling compacts and flushes for region at 1731482155755Disabling writes for close at 1731482155755Writing region close event to WAL at 1731482155755Closed at 1731482155755 2024-11-13T07:15:55,756 INFO [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => d029c66cacb99cb6bcc28a1f47b21331, NAME => 'SPTestTable,ccccccc8,1731482155666.d029c66cacb99cb6bcc28a1f47b21331.', STARTKEY => 'ccccccc8', ENDKEY => 'e6666661'}, tableDescriptor='SPTestTable', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca 2024-11-13T07:15:55,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741844_1020 (size=62) 2024-11-13T07:15:55,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741844_1020 (size=62) 2024-11-13T07:15:55,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741844_1020 (size=62) 2024-11-13T07:15:55,766 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(898): Instantiated SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:15:55,766 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1722): Closing dc0d3e1b8b1a5a44332ecc1f0118002b, disabling compactions & flushes 2024-11-13T07:15:55,766 INFO [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1755): Closing region SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b. 2024-11-13T07:15:55,766 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b. 2024-11-13T07:15:55,766 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b. after waiting 0 ms 2024-11-13T07:15:55,766 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b. 2024-11-13T07:15:55,767 INFO [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1973): Closed SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b. 2024-11-13T07:15:55,767 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1676): Region close journal for dc0d3e1b8b1a5a44332ecc1f0118002b: Waiting for close lock at 1731482155766Disabling compacts and flushes for region at 1731482155766Disabling writes for close at 1731482155766Writing region close event to WAL at 1731482155766Closed at 1731482155767 (+1 ms) 2024-11-13T07:15:55,767 INFO [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => dcdab597c439e333702f5a0384e4db9a, NAME => 'SPTestTable,e6666661,1731482155666.dcdab597c439e333702f5a0384e4db9a.', STARTKEY => 'e6666661', ENDKEY => ''}, tableDescriptor='SPTestTable', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca 2024-11-13T07:15:55,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T07:15:55,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741845_1021 (size=62) 2024-11-13T07:15:55,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741845_1021 (size=62) 2024-11-13T07:15:55,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741845_1021 (size=62) 2024-11-13T07:15:55,793 DEBUG [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(898): Instantiated SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:15:55,793 DEBUG [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(1722): Closing 56b34ea82217cf90ce43d525dd7e1a9d, disabling compactions & flushes 2024-11-13T07:15:55,793 INFO [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(1755): Closing region SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d. 2024-11-13T07:15:55,793 DEBUG [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d. 2024-11-13T07:15:55,793 DEBUG [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d. after waiting 0 ms 2024-11-13T07:15:55,793 DEBUG [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d. 2024-11-13T07:15:55,793 INFO [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(1973): Closed SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d. 2024-11-13T07:15:55,793 DEBUG [RegionOpenAndInit-SPTestTable-pool-0 {}] regionserver.HRegion(1676): Region close journal for 56b34ea82217cf90ce43d525dd7e1a9d: Waiting for close lock at 1731482155793Disabling compacts and flushes for region at 1731482155793Disabling writes for close at 1731482155793Writing region close event to WAL at 1731482155793Closed at 1731482155793 2024-11-13T07:15:55,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741847_1023 (size=62) 2024-11-13T07:15:55,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741847_1023 (size=62) 2024-11-13T07:15:55,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741847_1023 (size=62) 2024-11-13T07:15:55,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741846_1022 (size=54) 2024-11-13T07:15:55,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741846_1022 (size=54) 2024-11-13T07:15:55,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741846_1022 (size=54) 2024-11-13T07:15:55,812 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(898): Instantiated SPTestTable,e6666661,1731482155666.dcdab597c439e333702f5a0384e4db9a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:15:55,812 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1722): Closing dcdab597c439e333702f5a0384e4db9a, disabling compactions & flushes 2024-11-13T07:15:55,812 INFO [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1755): Closing region SPTestTable,e6666661,1731482155666.dcdab597c439e333702f5a0384e4db9a. 2024-11-13T07:15:55,812 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,e6666661,1731482155666.dcdab597c439e333702f5a0384e4db9a. 2024-11-13T07:15:55,812 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,e6666661,1731482155666.dcdab597c439e333702f5a0384e4db9a. after waiting 0 ms 2024-11-13T07:15:55,812 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,e6666661,1731482155666.dcdab597c439e333702f5a0384e4db9a. 2024-11-13T07:15:55,812 INFO [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1973): Closed SPTestTable,e6666661,1731482155666.dcdab597c439e333702f5a0384e4db9a. 2024-11-13T07:15:55,812 DEBUG [RegionOpenAndInit-SPTestTable-pool-1 {}] regionserver.HRegion(1676): Region close journal for dcdab597c439e333702f5a0384e4db9a: Waiting for close lock at 1731482155812Disabling compacts and flushes for region at 1731482155812Disabling writes for close at 1731482155812Writing region close event to WAL at 1731482155812Closed at 1731482155812 2024-11-13T07:15:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T07:15:56,202 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(898): Instantiated SPTestTable,ccccccc8,1731482155666.d029c66cacb99cb6bcc28a1f47b21331.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:15:56,202 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1722): Closing d029c66cacb99cb6bcc28a1f47b21331, disabling compactions & flushes 2024-11-13T07:15:56,202 INFO [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1755): Closing region SPTestTable,ccccccc8,1731482155666.d029c66cacb99cb6bcc28a1f47b21331. 2024-11-13T07:15:56,202 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,ccccccc8,1731482155666.d029c66cacb99cb6bcc28a1f47b21331. 2024-11-13T07:15:56,202 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,ccccccc8,1731482155666.d029c66cacb99cb6bcc28a1f47b21331. after waiting 0 ms 2024-11-13T07:15:56,202 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,ccccccc8,1731482155666.d029c66cacb99cb6bcc28a1f47b21331. 2024-11-13T07:15:56,202 INFO [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1973): Closed SPTestTable,ccccccc8,1731482155666.d029c66cacb99cb6bcc28a1f47b21331. 2024-11-13T07:15:56,202 DEBUG [RegionOpenAndInit-SPTestTable-pool-2 {}] regionserver.HRegion(1676): Region close journal for d029c66cacb99cb6bcc28a1f47b21331: Waiting for close lock at 1731482156202Disabling compacts and flushes for region at 1731482156202Disabling writes for close at 1731482156202Writing region close event to WAL at 1731482156202Closed at 1731482156202 2024-11-13T07:15:56,204 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=SPTestTable execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T07:15:56,204 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"SPTestTable,,1731482155666.2e4b72fe1d04f499cf9c51ce52c79100.","families":{"info":[{"qualifier":"regioninfo","vlen":53,"tag":[],"timestamp":"1731482156204"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731482156204"}]},"ts":"1731482156204"} 2024-11-13T07:15:56,204 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1731482156204"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731482156204"}]},"ts":"1731482156204"} 2024-11-13T07:15:56,205 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"SPTestTable,33333332,1731482155666.045378476123254a5c2616896d4edfcf.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1731482156204"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731482156204"}]},"ts":"1731482156204"} 2024-11-13T07:15:56,205 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"SPTestTable,66666664,1731482155666.1357138ab69f6578d72ce8d295f98b00.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1731482156204"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731482156204"}]},"ts":"1731482156204"} 2024-11-13T07:15:56,205 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1731482156204"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731482156204"}]},"ts":"1731482156204"} 2024-11-13T07:15:56,205 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"SPTestTable,7ffffffd,1731482155666.9e8af980aa626886ac3cb035f55a2e4c.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1731482156204"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731482156204"}]},"ts":"1731482156204"} 2024-11-13T07:15:56,205 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1731482156204"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731482156204"}]},"ts":"1731482156204"} 2024-11-13T07:15:56,205 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1731482156204"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731482156204"}]},"ts":"1731482156204"} 2024-11-13T07:15:56,205 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"SPTestTable,e6666661,1731482155666.dcdab597c439e333702f5a0384e4db9a.","families":{"info":[{"qualifier":"regioninfo","vlen":53,"tag":[],"timestamp":"1731482156204"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731482156204"}]},"ts":"1731482156204"} 2024-11-13T07:15:56,205 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"SPTestTable,ccccccc8,1731482155666.d029c66cacb99cb6bcc28a1f47b21331.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1731482156204"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731482156204"}]},"ts":"1731482156204"} 2024-11-13T07:15:56,210 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 10 regions to meta. 2024-11-13T07:15:56,211 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=SPTestTable execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T07:15:56,211 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"SPTestTable","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731482156211"}]},"ts":"1731482156211"} 2024-11-13T07:15:56,213 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=SPTestTable, state=ENABLING in hbase:meta 2024-11-13T07:15:56,214 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {2998b885bdef=0} racks are {/default-rack=0} 2024-11-13T07:15:56,219 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T07:15:56,219 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T07:15:56,219 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-13T07:15:56,219 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T07:15:56,219 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T07:15:56,219 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T07:15:56,219 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T07:15:56,219 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T07:15:56,219 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T07:15:56,219 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T07:15:56,220 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=2e4b72fe1d04f499cf9c51ce52c79100, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=1f49ca9e4320a113a985ca82c73f5c2a, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=045378476123254a5c2616896d4edfcf, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=ff8b864839731879b51af65fe2a46ac9, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=1357138ab69f6578d72ce8d295f98b00, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=9e8af980aa626886ac3cb035f55a2e4c, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=dc0d3e1b8b1a5a44332ecc1f0118002b, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=56b34ea82217cf90ce43d525dd7e1a9d, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=d029c66cacb99cb6bcc28a1f47b21331, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=dcdab597c439e333702f5a0384e4db9a, ASSIGN}] 2024-11-13T07:15:56,223 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=1f49ca9e4320a113a985ca82c73f5c2a, ASSIGN 2024-11-13T07:15:56,223 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=ff8b864839731879b51af65fe2a46ac9, ASSIGN 2024-11-13T07:15:56,223 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=1357138ab69f6578d72ce8d295f98b00, ASSIGN 2024-11-13T07:15:56,224 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=2e4b72fe1d04f499cf9c51ce52c79100, ASSIGN 2024-11-13T07:15:56,225 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=1357138ab69f6578d72ce8d295f98b00, ASSIGN; state=OFFLINE, location=2998b885bdef,41365,1731482143808; forceNewPlan=false, retain=false 2024-11-13T07:15:56,225 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=ff8b864839731879b51af65fe2a46ac9, ASSIGN; state=OFFLINE, location=2998b885bdef,39195,1731482143882; forceNewPlan=false, retain=false 2024-11-13T07:15:56,225 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=1f49ca9e4320a113a985ca82c73f5c2a, ASSIGN; state=OFFLINE, location=2998b885bdef,39195,1731482143882; forceNewPlan=false, retain=false 2024-11-13T07:15:56,226 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=2e4b72fe1d04f499cf9c51ce52c79100, ASSIGN; state=OFFLINE, location=2998b885bdef,41365,1731482143808; forceNewPlan=false, retain=false 2024-11-13T07:15:56,230 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=56b34ea82217cf90ce43d525dd7e1a9d, ASSIGN 2024-11-13T07:15:56,230 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=dcdab597c439e333702f5a0384e4db9a, ASSIGN 2024-11-13T07:15:56,232 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=56b34ea82217cf90ce43d525dd7e1a9d, ASSIGN; state=OFFLINE, location=2998b885bdef,39195,1731482143882; forceNewPlan=false, retain=false 2024-11-13T07:15:56,232 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=d029c66cacb99cb6bcc28a1f47b21331, ASSIGN 2024-11-13T07:15:56,232 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=dc0d3e1b8b1a5a44332ecc1f0118002b, ASSIGN 2024-11-13T07:15:56,234 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=dcdab597c439e333702f5a0384e4db9a, ASSIGN; state=OFFLINE, location=2998b885bdef,36543,1731482143970; forceNewPlan=false, retain=false 2024-11-13T07:15:56,234 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=dc0d3e1b8b1a5a44332ecc1f0118002b, ASSIGN; state=OFFLINE, location=2998b885bdef,41365,1731482143808; forceNewPlan=false, retain=false 2024-11-13T07:15:56,235 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=d029c66cacb99cb6bcc28a1f47b21331, ASSIGN; state=OFFLINE, location=2998b885bdef,41365,1731482143808; forceNewPlan=false, retain=false 2024-11-13T07:15:56,237 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=9e8af980aa626886ac3cb035f55a2e4c, ASSIGN 2024-11-13T07:15:56,238 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=045378476123254a5c2616896d4edfcf, ASSIGN 2024-11-13T07:15:56,239 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=9e8af980aa626886ac3cb035f55a2e4c, ASSIGN; state=OFFLINE, location=2998b885bdef,36543,1731482143970; forceNewPlan=false, retain=false 2024-11-13T07:15:56,239 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=045378476123254a5c2616896d4edfcf, ASSIGN; state=OFFLINE, location=2998b885bdef,36543,1731482143970; forceNewPlan=false, retain=false 2024-11-13T07:15:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T07:15:56,376 INFO [2998b885bdef:44607 {}] balancer.BaseLoadBalancer(388): Reassigned 10 regions. 10 retained the pre-restart assignment. 2024-11-13T07:15:56,377 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=d029c66cacb99cb6bcc28a1f47b21331, regionState=OPENING, regionLocation=2998b885bdef,41365,1731482143808 2024-11-13T07:15:56,377 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=dc0d3e1b8b1a5a44332ecc1f0118002b, regionState=OPENING, regionLocation=2998b885bdef,41365,1731482143808 2024-11-13T07:15:56,377 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=1357138ab69f6578d72ce8d295f98b00, regionState=OPENING, regionLocation=2998b885bdef,41365,1731482143808 2024-11-13T07:15:56,378 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=2e4b72fe1d04f499cf9c51ce52c79100, regionState=OPENING, regionLocation=2998b885bdef,41365,1731482143808 2024-11-13T07:15:56,379 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=56b34ea82217cf90ce43d525dd7e1a9d, regionState=OPENING, regionLocation=2998b885bdef,39195,1731482143882 2024-11-13T07:15:56,379 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ff8b864839731879b51af65fe2a46ac9, regionState=OPENING, regionLocation=2998b885bdef,39195,1731482143882 2024-11-13T07:15:56,379 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=dcdab597c439e333702f5a0384e4db9a, regionState=OPENING, regionLocation=2998b885bdef,36543,1731482143970 2024-11-13T07:15:56,379 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=1f49ca9e4320a113a985ca82c73f5c2a, regionState=OPENING, regionLocation=2998b885bdef,39195,1731482143882 2024-11-13T07:15:56,379 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=9e8af980aa626886ac3cb035f55a2e4c, regionState=OPENING, regionLocation=2998b885bdef,36543,1731482143970 2024-11-13T07:15:56,380 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=045378476123254a5c2616896d4edfcf, regionState=OPENING, regionLocation=2998b885bdef,36543,1731482143970 2024-11-13T07:15:56,383 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=dc0d3e1b8b1a5a44332ecc1f0118002b, ASSIGN because future has completed 2024-11-13T07:15:56,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc0d3e1b8b1a5a44332ecc1f0118002b, server=2998b885bdef,41365,1731482143808}] 2024-11-13T07:15:56,384 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=d029c66cacb99cb6bcc28a1f47b21331, ASSIGN because future has completed 2024-11-13T07:15:56,385 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure d029c66cacb99cb6bcc28a1f47b21331, server=2998b885bdef,41365,1731482143808}] 2024-11-13T07:15:56,385 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=1357138ab69f6578d72ce8d295f98b00, ASSIGN because future has completed 2024-11-13T07:15:56,386 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1357138ab69f6578d72ce8d295f98b00, server=2998b885bdef,41365,1731482143808}] 2024-11-13T07:15:56,387 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=2e4b72fe1d04f499cf9c51ce52c79100, ASSIGN because future has completed 2024-11-13T07:15:56,388 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=56b34ea82217cf90ce43d525dd7e1a9d, ASSIGN because future has completed 2024-11-13T07:15:56,389 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=dcdab597c439e333702f5a0384e4db9a, ASSIGN because future has completed 2024-11-13T07:15:56,390 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure dcdab597c439e333702f5a0384e4db9a, server=2998b885bdef,36543,1731482143970}] 2024-11-13T07:15:56,390 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56b34ea82217cf90ce43d525dd7e1a9d, server=2998b885bdef,39195,1731482143882}] 2024-11-13T07:15:56,391 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=ff8b864839731879b51af65fe2a46ac9, ASSIGN because future has completed 2024-11-13T07:15:56,392 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=9e8af980aa626886ac3cb035f55a2e4c, ASSIGN because future has completed 2024-11-13T07:15:56,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e4b72fe1d04f499cf9c51ce52c79100, server=2998b885bdef,41365,1731482143808}] 2024-11-13T07:15:56,393 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=1f49ca9e4320a113a985ca82c73f5c2a, ASSIGN because future has completed 2024-11-13T07:15:56,394 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1f49ca9e4320a113a985ca82c73f5c2a, server=2998b885bdef,39195,1731482143882}] 2024-11-13T07:15:56,394 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=045378476123254a5c2616896d4edfcf, ASSIGN because future has completed 2024-11-13T07:15:56,398 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 045378476123254a5c2616896d4edfcf, server=2998b885bdef,36543,1731482143970}] 2024-11-13T07:15:56,398 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9e8af980aa626886ac3cb035f55a2e4c, server=2998b885bdef,36543,1731482143970}] 2024-11-13T07:15:56,401 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff8b864839731879b51af65fe2a46ac9, server=2998b885bdef,39195,1731482143882}] 2024-11-13T07:15:56,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T07:15:57,111 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T07:15:57,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T07:15:59,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T07:16:00,473 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T07:16:00,490 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T07:16:00,490 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T07:16:00,999 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-13T07:16:01,002 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T07:16:01,033 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T07:16:01,033 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T07:16:03,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T07:16:06,387 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T07:16:06,388 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51873, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T07:16:06,400 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T07:16:06,413 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55367, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T07:16:06,420 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open SPTestTable,66666664,1731482155666.1357138ab69f6578d72ce8d295f98b00. 2024-11-13T07:16:06,420 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1357138ab69f6578d72ce8d295f98b00, NAME => 'SPTestTable,66666664,1731482155666.1357138ab69f6578d72ce8d295f98b00.', STARTKEY => '66666664', ENDKEY => '7ffffffd'} 2024-11-13T07:16:06,421 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table SPTestTable 1357138ab69f6578d72ce8d295f98b00 2024-11-13T07:16:06,421 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated SPTestTable,66666664,1731482155666.1357138ab69f6578d72ce8d295f98b00.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:16:06,421 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 1357138ab69f6578d72ce8d295f98b00 2024-11-13T07:16:06,421 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 1357138ab69f6578d72ce8d295f98b00 2024-11-13T07:16:06,436 INFO [StoreOpener-1357138ab69f6578d72ce8d295f98b00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region 1357138ab69f6578d72ce8d295f98b00 2024-11-13T07:16:06,444 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(132): Open SPTestTable,e6666661,1731482155666.dcdab597c439e333702f5a0384e4db9a. 2024-11-13T07:16:06,444 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7752): Opening region: {ENCODED => dcdab597c439e333702f5a0384e4db9a, NAME => 'SPTestTable,e6666661,1731482155666.dcdab597c439e333702f5a0384e4db9a.', STARTKEY => 'e6666661', ENDKEY => ''} 2024-11-13T07:16:06,445 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table SPTestTable dcdab597c439e333702f5a0384e4db9a 2024-11-13T07:16:06,445 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(898): Instantiated SPTestTable,e6666661,1731482155666.dcdab597c439e333702f5a0384e4db9a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:16:06,445 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7794): checking encryption for dcdab597c439e333702f5a0384e4db9a 2024-11-13T07:16:06,445 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(7797): checking classloading for dcdab597c439e333702f5a0384e4db9a 2024-11-13T07:16:06,449 INFO [StoreOpener-1357138ab69f6578d72ce8d295f98b00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1357138ab69f6578d72ce8d295f98b00 columnFamilyName cf 2024-11-13T07:16:06,449 DEBUG [StoreOpener-1357138ab69f6578d72ce8d295f98b00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:16:06,450 INFO [StoreOpener-1357138ab69f6578d72ce8d295f98b00-1 {}] regionserver.HStore(327): Store=1357138ab69f6578d72ce8d295f98b00/cf, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:16:06,450 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 1357138ab69f6578d72ce8d295f98b00 2024-11-13T07:16:06,452 INFO [StoreOpener-dcdab597c439e333702f5a0384e4db9a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region dcdab597c439e333702f5a0384e4db9a 2024-11-13T07:16:06,459 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a. 2024-11-13T07:16:06,459 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 1f49ca9e4320a113a985ca82c73f5c2a, NAME => 'SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a.', STARTKEY => '19999999', ENDKEY => '33333332'} 2024-11-13T07:16:06,459 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table SPTestTable 1f49ca9e4320a113a985ca82c73f5c2a 2024-11-13T07:16:06,459 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:16:06,459 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 1f49ca9e4320a113a985ca82c73f5c2a 2024-11-13T07:16:06,460 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 1f49ca9e4320a113a985ca82c73f5c2a 2024-11-13T07:16:06,463 INFO [StoreOpener-dcdab597c439e333702f5a0384e4db9a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcdab597c439e333702f5a0384e4db9a columnFamilyName cf 2024-11-13T07:16:06,463 DEBUG [StoreOpener-dcdab597c439e333702f5a0384e4db9a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:16:06,465 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/1357138ab69f6578d72ce8d295f98b00 2024-11-13T07:16:06,466 INFO [StoreOpener-dcdab597c439e333702f5a0384e4db9a-1 {}] regionserver.HStore(327): Store=dcdab597c439e333702f5a0384e4db9a/cf, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:16:06,466 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1038): replaying wal for dcdab597c439e333702f5a0384e4db9a 2024-11-13T07:16:06,467 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/dcdab597c439e333702f5a0384e4db9a 2024-11-13T07:16:06,468 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/dcdab597c439e333702f5a0384e4db9a 2024-11-13T07:16:06,468 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1048): stopping wal replay for dcdab597c439e333702f5a0384e4db9a 2024-11-13T07:16:06,468 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1060): Cleaning up temporary data for dcdab597c439e333702f5a0384e4db9a 2024-11-13T07:16:06,468 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/1357138ab69f6578d72ce8d295f98b00 2024-11-13T07:16:06,472 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1093): writing seq id for dcdab597c439e333702f5a0384e4db9a 2024-11-13T07:16:06,473 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1357138ab69f6578d72ce8d295f98b00 2024-11-13T07:16:06,473 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1357138ab69f6578d72ce8d295f98b00 2024-11-13T07:16:06,480 INFO [StoreOpener-1f49ca9e4320a113a985ca82c73f5c2a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region 1f49ca9e4320a113a985ca82c73f5c2a 2024-11-13T07:16:06,480 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/dcdab597c439e333702f5a0384e4db9a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:16:06,488 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1114): Opened dcdab597c439e333702f5a0384e4db9a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70685417, jitterRate=0.053294792771339417}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:16:06,488 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dcdab597c439e333702f5a0384e4db9a 2024-11-13T07:16:06,488 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegion(1006): Region open journal for dcdab597c439e333702f5a0384e4db9a: Running coprocessor pre-open hook at 1731482166445Writing region info on filesystem at 1731482166445Initializing all the Stores at 1731482166446 (+1 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482166446Cleaning up temporary data from old regions at 1731482166468 (+22 ms)Running coprocessor post-open hooks at 1731482166488 (+20 ms)Region opened successfully at 1731482166488 2024-11-13T07:16:06,492 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 1357138ab69f6578d72ce8d295f98b00 2024-11-13T07:16:06,492 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2236): Post open deploy tasks for SPTestTable,e6666661,1731482155666.dcdab597c439e333702f5a0384e4db9a., pid=20, masterSystemTime=1731482166401 2024-11-13T07:16:06,497 INFO [StoreOpener-1f49ca9e4320a113a985ca82c73f5c2a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f49ca9e4320a113a985ca82c73f5c2a columnFamilyName cf 2024-11-13T07:16:06,497 DEBUG [StoreOpener-1f49ca9e4320a113a985ca82c73f5c2a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:16:06,501 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] regionserver.HRegionServer(2266): Finished post open deploy task for SPTestTable,e6666661,1731482155666.dcdab597c439e333702f5a0384e4db9a. 2024-11-13T07:16:06,501 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=20}] handler.AssignRegionHandler(153): Opened SPTestTable,e6666661,1731482155666.dcdab597c439e333702f5a0384e4db9a. 2024-11-13T07:16:06,501 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] handler.AssignRegionHandler(132): Open SPTestTable,7ffffffd,1731482155666.9e8af980aa626886ac3cb035f55a2e4c. 2024-11-13T07:16:06,501 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(7752): Opening region: {ENCODED => 9e8af980aa626886ac3cb035f55a2e4c, NAME => 'SPTestTable,7ffffffd,1731482155666.9e8af980aa626886ac3cb035f55a2e4c.', STARTKEY => '7ffffffd', ENDKEY => '99999996'} 2024-11-13T07:16:06,502 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table SPTestTable 9e8af980aa626886ac3cb035f55a2e4c 2024-11-13T07:16:06,502 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(898): Instantiated SPTestTable,7ffffffd,1731482155666.9e8af980aa626886ac3cb035f55a2e4c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:16:06,502 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(7794): checking encryption for 9e8af980aa626886ac3cb035f55a2e4c 2024-11-13T07:16:06,502 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(7797): checking classloading for 9e8af980aa626886ac3cb035f55a2e4c 2024-11-13T07:16:06,502 INFO [StoreOpener-1f49ca9e4320a113a985ca82c73f5c2a-1 {}] regionserver.HStore(327): Store=1f49ca9e4320a113a985ca82c73f5c2a/cf, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:16:06,503 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 1f49ca9e4320a113a985ca82c73f5c2a 2024-11-13T07:16:06,509 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/1357138ab69f6578d72ce8d295f98b00/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:16:06,509 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/1f49ca9e4320a113a985ca82c73f5c2a 2024-11-13T07:16:06,509 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 1357138ab69f6578d72ce8d295f98b00; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59093053, jitterRate=-0.11944489181041718}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:16:06,509 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1357138ab69f6578d72ce8d295f98b00 2024-11-13T07:16:06,510 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 1357138ab69f6578d72ce8d295f98b00: Running coprocessor pre-open hook at 1731482166421Writing region info on filesystem at 1731482166421Initializing all the Stores at 1731482166431 (+10 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482166431Cleaning up temporary data from old regions at 1731482166473 (+42 ms)Running coprocessor post-open hooks at 1731482166509 (+36 ms)Region opened successfully at 1731482166510 (+1 ms) 2024-11-13T07:16:06,513 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/1f49ca9e4320a113a985ca82c73f5c2a 2024-11-13T07:16:06,516 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=dcdab597c439e333702f5a0384e4db9a, regionState=OPEN, openSeqNum=2, regionLocation=2998b885bdef,36543,1731482143970 2024-11-13T07:16:06,516 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 1f49ca9e4320a113a985ca82c73f5c2a 2024-11-13T07:16:06,517 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 1f49ca9e4320a113a985ca82c73f5c2a 2024-11-13T07:16:06,519 INFO [StoreOpener-9e8af980aa626886ac3cb035f55a2e4c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region 9e8af980aa626886ac3cb035f55a2e4c 2024-11-13T07:16:06,521 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for SPTestTable,66666664,1731482155666.1357138ab69f6578d72ce8d295f98b00., pid=17, masterSystemTime=1731482166386 2024-11-13T07:16:06,522 INFO [StoreOpener-9e8af980aa626886ac3cb035f55a2e4c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e8af980aa626886ac3cb035f55a2e4c columnFamilyName cf 2024-11-13T07:16:06,522 DEBUG [StoreOpener-9e8af980aa626886ac3cb035f55a2e4c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:16:06,523 INFO [StoreOpener-9e8af980aa626886ac3cb035f55a2e4c-1 {}] regionserver.HStore(327): Store=9e8af980aa626886ac3cb035f55a2e4c/cf, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:16:06,523 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1038): replaying wal for 9e8af980aa626886ac3cb035f55a2e4c 2024-11-13T07:16:06,524 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/9e8af980aa626886ac3cb035f55a2e4c 2024-11-13T07:16:06,524 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/9e8af980aa626886ac3cb035f55a2e4c 2024-11-13T07:16:06,525 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1048): stopping wal replay for 9e8af980aa626886ac3cb035f55a2e4c 2024-11-13T07:16:06,525 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1060): Cleaning up temporary data for 9e8af980aa626886ac3cb035f55a2e4c 2024-11-13T07:16:06,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure dcdab597c439e333702f5a0384e4db9a, server=2998b885bdef,36543,1731482143970 because future has completed 2024-11-13T07:16:06,533 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for SPTestTable,66666664,1731482155666.1357138ab69f6578d72ce8d295f98b00. 2024-11-13T07:16:06,533 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened SPTestTable,66666664,1731482155666.1357138ab69f6578d72ce8d295f98b00. 2024-11-13T07:16:06,533 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open SPTestTable,,1731482155666.2e4b72fe1d04f499cf9c51ce52c79100. 2024-11-13T07:16:06,533 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => 2e4b72fe1d04f499cf9c51ce52c79100, NAME => 'SPTestTable,,1731482155666.2e4b72fe1d04f499cf9c51ce52c79100.', STARTKEY => '', ENDKEY => '19999999'} 2024-11-13T07:16:06,534 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table SPTestTable 2e4b72fe1d04f499cf9c51ce52c79100 2024-11-13T07:16:06,534 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated SPTestTable,,1731482155666.2e4b72fe1d04f499cf9c51ce52c79100.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:16:06,534 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for 2e4b72fe1d04f499cf9c51ce52c79100 2024-11-13T07:16:06,534 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for 2e4b72fe1d04f499cf9c51ce52c79100 2024-11-13T07:16:06,541 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1093): writing seq id for 9e8af980aa626886ac3cb035f55a2e4c 2024-11-13T07:16:06,541 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 1f49ca9e4320a113a985ca82c73f5c2a 2024-11-13T07:16:06,544 INFO [StoreOpener-2e4b72fe1d04f499cf9c51ce52c79100-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region 2e4b72fe1d04f499cf9c51ce52c79100 2024-11-13T07:16:06,547 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=1357138ab69f6578d72ce8d295f98b00, regionState=OPEN, openSeqNum=2, regionLocation=2998b885bdef,41365,1731482143808 2024-11-13T07:16:06,550 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=14 2024-11-13T07:16:06,550 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure dcdab597c439e333702f5a0384e4db9a, server=2998b885bdef,36543,1731482143970 in 10.1420 sec 2024-11-13T07:16:06,554 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=dcdab597c439e333702f5a0384e4db9a, ASSIGN in 10.3300 sec 2024-11-13T07:16:06,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1357138ab69f6578d72ce8d295f98b00, server=2998b885bdef,41365,1731482143808 because future has completed 2024-11-13T07:16:06,559 INFO [StoreOpener-2e4b72fe1d04f499cf9c51ce52c79100-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2e4b72fe1d04f499cf9c51ce52c79100 columnFamilyName cf 2024-11-13T07:16:06,559 DEBUG [StoreOpener-2e4b72fe1d04f499cf9c51ce52c79100-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:16:06,569 INFO [StoreOpener-2e4b72fe1d04f499cf9c51ce52c79100-1 {}] regionserver.HStore(327): Store=2e4b72fe1d04f499cf9c51ce52c79100/cf, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:16:06,569 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for 2e4b72fe1d04f499cf9c51ce52c79100 2024-11-13T07:16:06,571 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/2e4b72fe1d04f499cf9c51ce52c79100 2024-11-13T07:16:06,572 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/2e4b72fe1d04f499cf9c51ce52c79100 2024-11-13T07:16:06,572 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/9e8af980aa626886ac3cb035f55a2e4c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:16:06,572 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for 2e4b72fe1d04f499cf9c51ce52c79100 2024-11-13T07:16:06,572 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/1f49ca9e4320a113a985ca82c73f5c2a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:16:06,573 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for 2e4b72fe1d04f499cf9c51ce52c79100 2024-11-13T07:16:06,573 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 1f49ca9e4320a113a985ca82c73f5c2a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72732218, jitterRate=0.083794504404068}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:16:06,574 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1f49ca9e4320a113a985ca82c73f5c2a 2024-11-13T07:16:06,574 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 1f49ca9e4320a113a985ca82c73f5c2a: Running coprocessor pre-open hook at 1731482166460Writing region info on filesystem at 1731482166460Initializing all the Stores at 1731482166476 (+16 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482166476Cleaning up temporary data from old regions at 1731482166517 (+41 ms)Running coprocessor post-open hooks at 1731482166574 (+57 ms)Region opened successfully at 1731482166574 2024-11-13T07:16:06,575 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=9 2024-11-13T07:16:06,575 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1114): Opened 9e8af980aa626886ac3cb035f55a2e4c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69276109, jitterRate=0.032294467091560364}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:16:06,575 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9e8af980aa626886ac3cb035f55a2e4c 2024-11-13T07:16:06,575 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 1357138ab69f6578d72ce8d295f98b00, server=2998b885bdef,41365,1731482143808 in 10.1730 sec 2024-11-13T07:16:06,575 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegion(1006): Region open journal for 9e8af980aa626886ac3cb035f55a2e4c: Running coprocessor pre-open hook at 1731482166502Writing region info on filesystem at 1731482166502Initializing all the Stores at 1731482166510 (+8 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482166510Cleaning up temporary data from old regions at 1731482166525 (+15 ms)Running coprocessor post-open hooks at 1731482166575 (+50 ms)Region opened successfully at 1731482166575 2024-11-13T07:16:06,576 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a., pid=21, masterSystemTime=1731482166400 2024-11-13T07:16:06,577 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegionServer(2236): Post open deploy tasks for SPTestTable,7ffffffd,1731482155666.9e8af980aa626886ac3cb035f55a2e4c., pid=22, masterSystemTime=1731482166401 2024-11-13T07:16:06,578 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for 2e4b72fe1d04f499cf9c51ce52c79100 2024-11-13T07:16:06,582 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=1357138ab69f6578d72ce8d295f98b00, ASSIGN in 10.3550 sec 2024-11-13T07:16:06,583 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/2e4b72fe1d04f499cf9c51ce52c79100/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:16:06,584 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened 2e4b72fe1d04f499cf9c51ce52c79100; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72524597, jitterRate=0.08070071041584015}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:16:06,584 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2e4b72fe1d04f499cf9c51ce52c79100 2024-11-13T07:16:06,584 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for 2e4b72fe1d04f499cf9c51ce52c79100: Running coprocessor pre-open hook at 1731482166534Writing region info on filesystem at 1731482166534Initializing all the Stores at 1731482166536 (+2 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482166536Cleaning up temporary data from old regions at 1731482166573 (+37 ms)Running coprocessor post-open hooks at 1731482166584 (+11 ms)Region opened successfully at 1731482166584 2024-11-13T07:16:06,585 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a. 2024-11-13T07:16:06,585 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a. 2024-11-13T07:16:06,586 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(132): Open SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d. 2024-11-13T07:16:06,586 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7752): Opening region: {ENCODED => 56b34ea82217cf90ce43d525dd7e1a9d, NAME => 'SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d.', STARTKEY => 'b333332f', ENDKEY => 'ccccccc8'} 2024-11-13T07:16:06,586 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table SPTestTable 56b34ea82217cf90ce43d525dd7e1a9d 2024-11-13T07:16:06,586 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(898): Instantiated SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:16:06,586 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7794): checking encryption for 56b34ea82217cf90ce43d525dd7e1a9d 2024-11-13T07:16:06,586 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(7797): checking classloading for 56b34ea82217cf90ce43d525dd7e1a9d 2024-11-13T07:16:06,590 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=1f49ca9e4320a113a985ca82c73f5c2a, regionState=OPEN, openSeqNum=2, regionLocation=2998b885bdef,39195,1731482143882 2024-11-13T07:16:06,591 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] regionserver.HRegionServer(2266): Finished post open deploy task for SPTestTable,7ffffffd,1731482155666.9e8af980aa626886ac3cb035f55a2e4c. 2024-11-13T07:16:06,591 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=22}] handler.AssignRegionHandler(153): Opened SPTestTable,7ffffffd,1731482155666.9e8af980aa626886ac3cb035f55a2e4c. 2024-11-13T07:16:06,591 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(132): Open SPTestTable,33333332,1731482155666.045378476123254a5c2616896d4edfcf. 2024-11-13T07:16:06,591 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7752): Opening region: {ENCODED => 045378476123254a5c2616896d4edfcf, NAME => 'SPTestTable,33333332,1731482155666.045378476123254a5c2616896d4edfcf.', STARTKEY => '33333332', ENDKEY => '4ccccccb'} 2024-11-13T07:16:06,592 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for SPTestTable,,1731482155666.2e4b72fe1d04f499cf9c51ce52c79100., pid=18, masterSystemTime=1731482166386 2024-11-13T07:16:06,592 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table SPTestTable 045378476123254a5c2616896d4edfcf 2024-11-13T07:16:06,592 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(898): Instantiated SPTestTable,33333332,1731482155666.045378476123254a5c2616896d4edfcf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:16:06,592 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7794): checking encryption for 045378476123254a5c2616896d4edfcf 2024-11-13T07:16:06,592 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(7797): checking classloading for 045378476123254a5c2616896d4edfcf 2024-11-13T07:16:06,592 INFO [StoreOpener-56b34ea82217cf90ce43d525dd7e1a9d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region 56b34ea82217cf90ce43d525dd7e1a9d 2024-11-13T07:16:06,593 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=9e8af980aa626886ac3cb035f55a2e4c, regionState=OPEN, openSeqNum=2, regionLocation=2998b885bdef,36543,1731482143970 2024-11-13T07:16:06,594 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1f49ca9e4320a113a985ca82c73f5c2a, server=2998b885bdef,39195,1731482143882 because future has completed 2024-11-13T07:16:06,603 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for SPTestTable,,1731482155666.2e4b72fe1d04f499cf9c51ce52c79100. 2024-11-13T07:16:06,603 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened SPTestTable,,1731482155666.2e4b72fe1d04f499cf9c51ce52c79100. 2024-11-13T07:16:06,603 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(132): Open SPTestTable,ccccccc8,1731482155666.d029c66cacb99cb6bcc28a1f47b21331. 2024-11-13T07:16:06,603 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7752): Opening region: {ENCODED => d029c66cacb99cb6bcc28a1f47b21331, NAME => 'SPTestTable,ccccccc8,1731482155666.d029c66cacb99cb6bcc28a1f47b21331.', STARTKEY => 'ccccccc8', ENDKEY => 'e6666661'} 2024-11-13T07:16:06,603 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9e8af980aa626886ac3cb035f55a2e4c, server=2998b885bdef,36543,1731482143970 because future has completed 2024-11-13T07:16:06,604 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table SPTestTable d029c66cacb99cb6bcc28a1f47b21331 2024-11-13T07:16:06,604 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(898): Instantiated SPTestTable,ccccccc8,1731482155666.d029c66cacb99cb6bcc28a1f47b21331.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:16:06,604 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7794): checking encryption for d029c66cacb99cb6bcc28a1f47b21331 2024-11-13T07:16:06,604 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(7797): checking classloading for d029c66cacb99cb6bcc28a1f47b21331 2024-11-13T07:16:06,605 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44607 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=2998b885bdef,36543,1731482143970, table=SPTestTable, region=9e8af980aa626886ac3cb035f55a2e4c. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T07:16:06,609 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44607 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=2998b885bdef,39195,1731482143882, table=SPTestTable, region=1f49ca9e4320a113a985ca82c73f5c2a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T07:16:06,610 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=2e4b72fe1d04f499cf9c51ce52c79100, regionState=OPEN, openSeqNum=2, regionLocation=2998b885bdef,41365,1731482143808 2024-11-13T07:16:06,611 INFO [StoreOpener-d029c66cacb99cb6bcc28a1f47b21331-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region d029c66cacb99cb6bcc28a1f47b21331 2024-11-13T07:16:06,612 INFO [StoreOpener-045378476123254a5c2616896d4edfcf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region 045378476123254a5c2616896d4edfcf 2024-11-13T07:16:06,613 INFO [StoreOpener-56b34ea82217cf90ce43d525dd7e1a9d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56b34ea82217cf90ce43d525dd7e1a9d columnFamilyName cf 2024-11-13T07:16:06,613 DEBUG [StoreOpener-56b34ea82217cf90ce43d525dd7e1a9d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:16:06,616 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2e4b72fe1d04f499cf9c51ce52c79100, server=2998b885bdef,41365,1731482143808 because future has completed 2024-11-13T07:16:06,617 INFO [StoreOpener-d029c66cacb99cb6bcc28a1f47b21331-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d029c66cacb99cb6bcc28a1f47b21331 columnFamilyName cf 2024-11-13T07:16:06,617 DEBUG [StoreOpener-d029c66cacb99cb6bcc28a1f47b21331-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:16:06,619 INFO [StoreOpener-56b34ea82217cf90ce43d525dd7e1a9d-1 {}] regionserver.HStore(327): Store=56b34ea82217cf90ce43d525dd7e1a9d/cf, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:16:06,620 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1038): replaying wal for 56b34ea82217cf90ce43d525dd7e1a9d 2024-11-13T07:16:06,620 INFO [StoreOpener-d029c66cacb99cb6bcc28a1f47b21331-1 {}] regionserver.HStore(327): Store=d029c66cacb99cb6bcc28a1f47b21331/cf, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:16:06,620 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1038): replaying wal for d029c66cacb99cb6bcc28a1f47b21331 2024-11-13T07:16:06,620 INFO [StoreOpener-045378476123254a5c2616896d4edfcf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 045378476123254a5c2616896d4edfcf columnFamilyName cf 2024-11-13T07:16:06,621 DEBUG [StoreOpener-045378476123254a5c2616896d4edfcf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:16:06,625 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/d029c66cacb99cb6bcc28a1f47b21331 2024-11-13T07:16:06,626 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/d029c66cacb99cb6bcc28a1f47b21331 2024-11-13T07:16:06,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=6 2024-11-13T07:16:06,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 1f49ca9e4320a113a985ca82c73f5c2a, server=2998b885bdef,39195,1731482143882 in 10.2150 sec 2024-11-13T07:16:06,627 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/56b34ea82217cf90ce43d525dd7e1a9d 2024-11-13T07:16:06,628 INFO [StoreOpener-045378476123254a5c2616896d4edfcf-1 {}] regionserver.HStore(327): Store=045378476123254a5c2616896d4edfcf/cf, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:16:06,629 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=10 2024-11-13T07:16:06,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 9e8af980aa626886ac3cb035f55a2e4c, server=2998b885bdef,36543,1731482143970 in 10.2150 sec 2024-11-13T07:16:06,631 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=1f49ca9e4320a113a985ca82c73f5c2a, ASSIGN in 10.4060 sec 2024-11-13T07:16:06,632 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1038): replaying wal for 045378476123254a5c2616896d4edfcf 2024-11-13T07:16:06,633 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/56b34ea82217cf90ce43d525dd7e1a9d 2024-11-13T07:16:06,633 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1048): stopping wal replay for d029c66cacb99cb6bcc28a1f47b21331 2024-11-13T07:16:06,633 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1060): Cleaning up temporary data for d029c66cacb99cb6bcc28a1f47b21331 2024-11-13T07:16:06,633 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/045378476123254a5c2616896d4edfcf 2024-11-13T07:16:06,638 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/045378476123254a5c2616896d4edfcf 2024-11-13T07:16:06,638 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1048): stopping wal replay for 56b34ea82217cf90ce43d525dd7e1a9d 2024-11-13T07:16:06,639 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1060): Cleaning up temporary data for 56b34ea82217cf90ce43d525dd7e1a9d 2024-11-13T07:16:06,641 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1093): writing seq id for d029c66cacb99cb6bcc28a1f47b21331 2024-11-13T07:16:06,645 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=5 2024-11-13T07:16:06,645 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 2e4b72fe1d04f499cf9c51ce52c79100, server=2998b885bdef,41365,1731482143808 in 10.2390 sec 2024-11-13T07:16:06,648 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=9e8af980aa626886ac3cb035f55a2e4c, ASSIGN in 10.4110 sec 2024-11-13T07:16:06,649 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=2e4b72fe1d04f499cf9c51ce52c79100, ASSIGN in 10.4250 sec 2024-11-13T07:16:06,649 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1048): stopping wal replay for 045378476123254a5c2616896d4edfcf 2024-11-13T07:16:06,649 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1060): Cleaning up temporary data for 045378476123254a5c2616896d4edfcf 2024-11-13T07:16:06,661 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1093): writing seq id for 045378476123254a5c2616896d4edfcf 2024-11-13T07:16:06,662 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1093): writing seq id for 56b34ea82217cf90ce43d525dd7e1a9d 2024-11-13T07:16:06,672 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/d029c66cacb99cb6bcc28a1f47b21331/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:16:06,674 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1114): Opened d029c66cacb99cb6bcc28a1f47b21331; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72586980, jitterRate=0.08163028955459595}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:16:06,675 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d029c66cacb99cb6bcc28a1f47b21331 2024-11-13T07:16:06,675 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegion(1006): Region open journal for d029c66cacb99cb6bcc28a1f47b21331: Running coprocessor pre-open hook at 1731482166604Writing region info on filesystem at 1731482166604Initializing all the Stores at 1731482166609 (+5 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482166609Cleaning up temporary data from old regions at 1731482166633 (+24 ms)Running coprocessor post-open hooks at 1731482166675 (+42 ms)Region opened successfully at 1731482166675 2024-11-13T07:16:06,676 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2236): Post open deploy tasks for SPTestTable,ccccccc8,1731482155666.d029c66cacb99cb6bcc28a1f47b21331., pid=16, masterSystemTime=1731482166386 2024-11-13T07:16:06,680 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/045378476123254a5c2616896d4edfcf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:16:06,682 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1114): Opened 045378476123254a5c2616896d4edfcf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68296575, jitterRate=0.017698273062705994}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:16:06,682 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 045378476123254a5c2616896d4edfcf 2024-11-13T07:16:06,682 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegion(1006): Region open journal for 045378476123254a5c2616896d4edfcf: Running coprocessor pre-open hook at 1731482166592Writing region info on filesystem at 1731482166592Initializing all the Stores at 1731482166604 (+12 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482166605 (+1 ms)Cleaning up temporary data from old regions at 1731482166649 (+44 ms)Running coprocessor post-open hooks at 1731482166682 (+33 ms)Region opened successfully at 1731482166682 2024-11-13T07:16:06,683 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] regionserver.HRegionServer(2266): Finished post open deploy task for SPTestTable,ccccccc8,1731482155666.d029c66cacb99cb6bcc28a1f47b21331. 2024-11-13T07:16:06,683 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=16}] handler.AssignRegionHandler(153): Opened SPTestTable,ccccccc8,1731482155666.d029c66cacb99cb6bcc28a1f47b21331. 2024-11-13T07:16:06,683 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b. 2024-11-13T07:16:06,683 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => dc0d3e1b8b1a5a44332ecc1f0118002b, NAME => 'SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b.', STARTKEY => '99999996', ENDKEY => 'b333332f'} 2024-11-13T07:16:06,683 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table SPTestTable dc0d3e1b8b1a5a44332ecc1f0118002b 2024-11-13T07:16:06,683 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:16:06,683 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for dc0d3e1b8b1a5a44332ecc1f0118002b 2024-11-13T07:16:06,683 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for dc0d3e1b8b1a5a44332ecc1f0118002b 2024-11-13T07:16:06,684 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=d029c66cacb99cb6bcc28a1f47b21331, regionState=OPEN, openSeqNum=2, regionLocation=2998b885bdef,41365,1731482143808 2024-11-13T07:16:06,685 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2236): Post open deploy tasks for SPTestTable,33333332,1731482155666.045378476123254a5c2616896d4edfcf., pid=23, masterSystemTime=1731482166401 2024-11-13T07:16:06,687 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/56b34ea82217cf90ce43d525dd7e1a9d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:16:06,687 INFO [StoreOpener-dc0d3e1b8b1a5a44332ecc1f0118002b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region dc0d3e1b8b1a5a44332ecc1f0118002b 2024-11-13T07:16:06,688 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1114): Opened 56b34ea82217cf90ce43d525dd7e1a9d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71922104, jitterRate=0.07172286510467529}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:16:06,688 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 56b34ea82217cf90ce43d525dd7e1a9d 2024-11-13T07:16:06,689 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegion(1006): Region open journal for 56b34ea82217cf90ce43d525dd7e1a9d: Running coprocessor pre-open hook at 1731482166586Writing region info on filesystem at 1731482166586Initializing all the Stores at 1731482166588 (+2 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482166588Cleaning up temporary data from old regions at 1731482166639 (+51 ms)Running coprocessor post-open hooks at 1731482166688 (+49 ms)Region opened successfully at 1731482166688 2024-11-13T07:16:06,691 INFO [StoreOpener-dc0d3e1b8b1a5a44332ecc1f0118002b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc0d3e1b8b1a5a44332ecc1f0118002b columnFamilyName cf 2024-11-13T07:16:06,691 DEBUG [StoreOpener-dc0d3e1b8b1a5a44332ecc1f0118002b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:16:06,692 INFO [StoreOpener-dc0d3e1b8b1a5a44332ecc1f0118002b-1 {}] regionserver.HStore(327): Store=dc0d3e1b8b1a5a44332ecc1f0118002b/cf, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:16:06,692 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for dc0d3e1b8b1a5a44332ecc1f0118002b 2024-11-13T07:16:06,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure d029c66cacb99cb6bcc28a1f47b21331, server=2998b885bdef,41365,1731482143808 because future has completed 2024-11-13T07:16:06,694 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/dc0d3e1b8b1a5a44332ecc1f0118002b 2024-11-13T07:16:06,694 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/dc0d3e1b8b1a5a44332ecc1f0118002b 2024-11-13T07:16:06,695 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for dc0d3e1b8b1a5a44332ecc1f0118002b 2024-11-13T07:16:06,695 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for dc0d3e1b8b1a5a44332ecc1f0118002b 2024-11-13T07:16:06,699 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for dc0d3e1b8b1a5a44332ecc1f0118002b 2024-11-13T07:16:06,699 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2236): Post open deploy tasks for SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d., pid=19, masterSystemTime=1731482166400 2024-11-13T07:16:06,700 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] regionserver.HRegionServer(2266): Finished post open deploy task for SPTestTable,33333332,1731482155666.045378476123254a5c2616896d4edfcf. 2024-11-13T07:16:06,701 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=23}] handler.AssignRegionHandler(153): Opened SPTestTable,33333332,1731482155666.045378476123254a5c2616896d4edfcf. 2024-11-13T07:16:06,702 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=045378476123254a5c2616896d4edfcf, regionState=OPEN, openSeqNum=2, regionLocation=2998b885bdef,36543,1731482143970 2024-11-13T07:16:06,706 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] regionserver.HRegionServer(2266): Finished post open deploy task for SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d. 2024-11-13T07:16:06,706 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=19}] handler.AssignRegionHandler(153): Opened SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d. 2024-11-13T07:16:06,706 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9. 2024-11-13T07:16:06,706 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => ff8b864839731879b51af65fe2a46ac9, NAME => 'SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9.', STARTKEY => '4ccccccb', ENDKEY => '66666664'} 2024-11-13T07:16:06,707 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 045378476123254a5c2616896d4edfcf, server=2998b885bdef,36543,1731482143970 because future has completed 2024-11-13T07:16:06,707 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table SPTestTable ff8b864839731879b51af65fe2a46ac9 2024-11-13T07:16:06,707 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:16:06,707 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for ff8b864839731879b51af65fe2a46ac9 2024-11-13T07:16:06,707 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for ff8b864839731879b51af65fe2a46ac9 2024-11-13T07:16:06,708 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=56b34ea82217cf90ce43d525dd7e1a9d, regionState=OPEN, openSeqNum=2, regionLocation=2998b885bdef,39195,1731482143882 2024-11-13T07:16:06,709 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-13T07:16:06,709 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure d029c66cacb99cb6bcc28a1f47b21331, server=2998b885bdef,41365,1731482143808 in 10.3150 sec 2024-11-13T07:16:06,709 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44607 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=2998b885bdef,36543,1731482143970, table=SPTestTable, region=045378476123254a5c2616896d4edfcf. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T07:16:06,713 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/dc0d3e1b8b1a5a44332ecc1f0118002b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:16:06,714 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened dc0d3e1b8b1a5a44332ecc1f0118002b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74839908, jitterRate=0.11520153284072876}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:16:06,714 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dc0d3e1b8b1a5a44332ecc1f0118002b 2024-11-13T07:16:06,715 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for dc0d3e1b8b1a5a44332ecc1f0118002b: Running coprocessor pre-open hook at 1731482166684Writing region info on filesystem at 1731482166684Initializing all the Stores at 1731482166686 (+2 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482166686Cleaning up temporary data from old regions at 1731482166695 (+9 ms)Running coprocessor post-open hooks at 1731482166714 (+19 ms)Region opened successfully at 1731482166714 2024-11-13T07:16:06,715 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44607 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=2998b885bdef,39195,1731482143882, table=SPTestTable, region=56b34ea82217cf90ce43d525dd7e1a9d. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T07:16:06,716 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b., pid=15, masterSystemTime=1731482166386 2024-11-13T07:16:06,717 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=d029c66cacb99cb6bcc28a1f47b21331, ASSIGN in 10.4890 sec 2024-11-13T07:16:06,718 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=7 2024-11-13T07:16:06,719 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 045378476123254a5c2616896d4edfcf, server=2998b885bdef,36543,1731482143970 in 10.3120 sec 2024-11-13T07:16:06,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56b34ea82217cf90ce43d525dd7e1a9d, server=2998b885bdef,39195,1731482143882 because future has completed 2024-11-13T07:16:06,725 INFO [StoreOpener-ff8b864839731879b51af65fe2a46ac9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region ff8b864839731879b51af65fe2a46ac9 2024-11-13T07:16:06,727 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b. 2024-11-13T07:16:06,727 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b. 2024-11-13T07:16:06,728 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=045378476123254a5c2616896d4edfcf, ASSIGN in 10.4990 sec 2024-11-13T07:16:06,730 INFO [StoreOpener-ff8b864839731879b51af65fe2a46ac9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff8b864839731879b51af65fe2a46ac9 columnFamilyName cf 2024-11-13T07:16:06,730 DEBUG [StoreOpener-ff8b864839731879b51af65fe2a46ac9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:16:06,730 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=dc0d3e1b8b1a5a44332ecc1f0118002b, regionState=OPEN, openSeqNum=2, regionLocation=2998b885bdef,41365,1731482143808 2024-11-13T07:16:06,731 INFO [StoreOpener-ff8b864839731879b51af65fe2a46ac9-1 {}] regionserver.HStore(327): Store=ff8b864839731879b51af65fe2a46ac9/cf, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:16:06,731 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for ff8b864839731879b51af65fe2a46ac9 2024-11-13T07:16:06,733 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/ff8b864839731879b51af65fe2a46ac9 2024-11-13T07:16:06,734 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/ff8b864839731879b51af65fe2a46ac9 2024-11-13T07:16:06,734 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=12 2024-11-13T07:16:06,734 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 56b34ea82217cf90ce43d525dd7e1a9d, server=2998b885bdef,39195,1731482143882 in 10.3370 sec 2024-11-13T07:16:06,735 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for ff8b864839731879b51af65fe2a46ac9 2024-11-13T07:16:06,735 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for ff8b864839731879b51af65fe2a46ac9 2024-11-13T07:16:06,741 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for ff8b864839731879b51af65fe2a46ac9 2024-11-13T07:16:06,741 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc0d3e1b8b1a5a44332ecc1f0118002b, server=2998b885bdef,41365,1731482143808 because future has completed 2024-11-13T07:16:06,751 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=56b34ea82217cf90ce43d525dd7e1a9d, ASSIGN in 10.5140 sec 2024-11-13T07:16:06,760 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/ff8b864839731879b51af65fe2a46ac9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T07:16:06,763 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened ff8b864839731879b51af65fe2a46ac9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60625286, jitterRate=-0.0966128408908844}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:16:06,763 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ff8b864839731879b51af65fe2a46ac9 2024-11-13T07:16:06,763 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for ff8b864839731879b51af65fe2a46ac9: Running coprocessor pre-open hook at 1731482166707Writing region info on filesystem at 1731482166707Initializing all the Stores at 1731482166712 (+5 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482166712Cleaning up temporary data from old regions at 1731482166735 (+23 ms)Running coprocessor post-open hooks at 1731482166763 (+28 ms)Region opened successfully at 1731482166763 2024-11-13T07:16:06,766 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=11 2024-11-13T07:16:06,767 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure dc0d3e1b8b1a5a44332ecc1f0118002b, server=2998b885bdef,41365,1731482143808 in 10.3670 sec 2024-11-13T07:16:06,770 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9., pid=24, masterSystemTime=1731482166400 2024-11-13T07:16:06,770 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=dc0d3e1b8b1a5a44332ecc1f0118002b, ASSIGN in 10.5470 sec 2024-11-13T07:16:06,788 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9. 2024-11-13T07:16:06,788 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9. 2024-11-13T07:16:06,793 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ff8b864839731879b51af65fe2a46ac9, regionState=OPEN, openSeqNum=2, regionLocation=2998b885bdef,39195,1731482143882 2024-11-13T07:16:06,797 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff8b864839731879b51af65fe2a46ac9, server=2998b885bdef,39195,1731482143882 because future has completed 2024-11-13T07:16:06,813 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=8 2024-11-13T07:16:06,816 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure ff8b864839731879b51af65fe2a46ac9, server=2998b885bdef,39195,1731482143882 in 10.3990 sec 2024-11-13T07:16:06,819 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=4 2024-11-13T07:16:06,819 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=ff8b864839731879b51af65fe2a46ac9, ASSIGN in 10.5930 sec 2024-11-13T07:16:06,821 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=SPTestTable execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T07:16:06,821 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"SPTestTable","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731482166821"}]},"ts":"1731482166821"} 2024-11-13T07:16:06,824 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=SPTestTable, state=ENABLED in hbase:meta 2024-11-13T07:16:06,827 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=SPTestTable execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T07:16:06,831 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=SPTestTable in 11.1600 sec 2024-11-13T07:16:09,113 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'SPTestTable' 2024-11-13T07:16:11,422 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T07:16:11,458 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T07:16:11,459 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T07:16:11,968 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-13T07:16:11,971 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T07:16:12,031 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T07:16:12,032 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-13T07:16:13,484 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T07:16:13,671 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=2998b885bdef:46239 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 10 more 2024-11-13T07:16:13,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T07:16:13,952 INFO [RPCClient-NioEventLoopGroup-6-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:SPTestTable completed 2024-11-13T07:16:13,952 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table SPTestTable get assigned. Timeout = 60000ms 2024-11-13T07:16:13,953 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T07:16:13,957 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table SPTestTable assigned to meta. Checking AM states. 2024-11-13T07:16:13,958 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T07:16:13,958 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table SPTestTable assigned. 2024-11-13T07:16:13,991 DEBUG [RPCClient-NioEventLoopGroup-6-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='aaa', locateType=CURRENT is [region=SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b., hostname=2998b885bdef,41365,1731482143808, seqNum=2] 2024-11-13T07:16:13,992 DEBUG [RPCClient-NioEventLoopGroup-6-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='aab', locateType=CURRENT is [region=SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b., hostname=2998b885bdef,41365,1731482143808, seqNum=2] 2024-11-13T07:16:13,994 DEBUG [RPCClient-NioEventLoopGroup-6-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='aac', locateType=CURRENT is [region=SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b., hostname=2998b885bdef,41365,1731482143808, seqNum=2] 2024-11-13T07:16:13,997 DEBUG [RPCClient-NioEventLoopGroup-6-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='aae', locateType=CURRENT is [region=SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b., hostname=2998b885bdef,41365,1731482143808, seqNum=2] 2024-11-13T07:16:13,999 DEBUG [RPCClient-NioEventLoopGroup-6-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='aad', locateType=CURRENT is [region=SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b., hostname=2998b885bdef,41365,1731482143808, seqNum=2] 2024-11-13T07:16:14,000 DEBUG [RPCClient-NioEventLoopGroup-6-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='aaf', locateType=CURRENT is [region=SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b., hostname=2998b885bdef,41365,1731482143808, seqNum=2] 2024-11-13T07:16:14,063 DEBUG [RPCClient-NioEventLoopGroup-6-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='baa', locateType=CURRENT is [region=SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d., hostname=2998b885bdef,39195,1731482143882, seqNum=2] 2024-11-13T07:16:14,064 DEBUG [RPCClient-NioEventLoopGroup-6-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='bab', locateType=CURRENT is [region=SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d., hostname=2998b885bdef,39195,1731482143882, seqNum=2] 2024-11-13T07:16:14,066 DEBUG [RPCClient-NioEventLoopGroup-6-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='bac', locateType=CURRENT is [region=SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d., hostname=2998b885bdef,39195,1731482143882, seqNum=2] 2024-11-13T07:16:14,072 DEBUG [RPCClient-NioEventLoopGroup-6-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='bad', locateType=CURRENT is [region=SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d., hostname=2998b885bdef,39195,1731482143882, seqNum=2] 2024-11-13T07:16:14,137 DEBUG [RPCClient-NioEventLoopGroup-6-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='ccd', locateType=CURRENT is [region=SPTestTable,ccccccc8,1731482155666.d029c66cacb99cb6bcc28a1f47b21331., hostname=2998b885bdef,41365,1731482143808, seqNum=2] 2024-11-13T07:16:14,147 DEBUG [RPCClient-NioEventLoopGroup-6-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='cce', locateType=CURRENT is [region=SPTestTable,ccccccc8,1731482155666.d029c66cacb99cb6bcc28a1f47b21331., hostname=2998b885bdef,41365,1731482143808, seqNum=2] 2024-11-13T07:16:14,150 DEBUG [RPCClient-NioEventLoopGroup-6-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='ccf', locateType=CURRENT is [region=SPTestTable,ccccccc8,1731482155666.d029c66cacb99cb6bcc28a1f47b21331., hostname=2998b885bdef,41365,1731482143808, seqNum=2] 2024-11-13T07:16:14,281 DEBUG [RPCClient-NioEventLoopGroup-6-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='eaa', locateType=CURRENT is [region=SPTestTable,e6666661,1731482155666.dcdab597c439e333702f5a0384e4db9a., hostname=2998b885bdef,36543,1731482143970, seqNum=2] 2024-11-13T07:16:14,283 DEBUG [RPCClient-NioEventLoopGroup-6-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='eab', locateType=CURRENT is [region=SPTestTable,e6666661,1731482155666.dcdab597c439e333702f5a0384e4db9a., hostname=2998b885bdef,36543,1731482143970, seqNum=2] 2024-11-13T07:16:14,293 DEBUG [RPCClient-NioEventLoopGroup-6-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'SPTestTable', row='eac', locateType=CURRENT is [region=SPTestTable,e6666661,1731482155666.dcdab597c439e333702f5a0384e4db9a., hostname=2998b885bdef,36543,1731482143970, seqNum=2] 2024-11-13T07:16:15,542 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T07:16:15,563 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T07:16:15,573 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T07:16:15,589 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T07:16:15,640 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41365 {}] regionserver.HRegion(8528): writing data to region SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b. with WAL disabled. Data may be lost in the event of a crash. 2024-11-13T07:16:15,663 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39195 {}] regionserver.HRegion(8528): writing data to region SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d. with WAL disabled. Data may be lost in the event of a crash. 2024-11-13T07:16:15,727 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41365 {}] regionserver.HRegion(8528): writing data to region SPTestTable,ccccccc8,1731482155666.d029c66cacb99cb6bcc28a1f47b21331. with WAL disabled. Data may be lost in the event of a crash. 2024-11-13T07:16:15,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] regionserver.RSRpcServices(2610): Large batch operation detected (greater than 5000) (HBASE-18023). Requested Number of Rows: 14872 Client: jenkins//172.17.0.3 first region in multi=SPTestTable,e6666661,1731482155666.dcdab597c439e333702f5a0384e4db9a. 2024-11-13T07:16:16,559 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36543 {}] regionserver.HRegion(8528): writing data to region SPTestTable,e6666661,1731482155666.dcdab597c439e333702f5a0384e4db9a. with WAL disabled. Data may be lost in the event of a crash. 2024-11-13T07:16:16,873 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=25, snapshot={ ss=SnapshotProcedureTest table=SPTestTable type=FLUSH ttl=0 } 2024-11-13T07:16:16,873 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T07:16:16,875 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=25, state=RUNNABLE:SNAPSHOT_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=25, snapshot={ ss=SnapshotProcedureTest table=SPTestTable type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2024-11-13T07:16:16,878 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=25, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=25, snapshot={ ss=SnapshotProcedureTest table=SPTestTable type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2024-11-13T07:16:16,888 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=25, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=25, snapshot={ ss=SnapshotProcedureTest table=SPTestTable type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2024-11-13T07:16:16,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741848_1024 (size=49) 2024-11-13T07:16:16,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741848_1024 (size=49) 2024-11-13T07:16:16,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741848_1024 (size=49) 2024-11-13T07:16:16,983 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=25, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=25, snapshot={ ss=SnapshotProcedureTest table=SPTestTable type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2024-11-13T07:16:16,984 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 2e4b72fe1d04f499cf9c51ce52c79100}, {pid=27, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 1f49ca9e4320a113a985ca82c73f5c2a}, {pid=28, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 045378476123254a5c2616896d4edfcf}, {pid=29, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure ff8b864839731879b51af65fe2a46ac9}, {pid=30, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 1357138ab69f6578d72ce8d295f98b00}, {pid=31, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9e8af980aa626886ac3cb035f55a2e4c}, {pid=32, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure dc0d3e1b8b1a5a44332ecc1f0118002b}, {pid=33, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 56b34ea82217cf90ce43d525dd7e1a9d}, {pid=34, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d029c66cacb99cb6bcc28a1f47b21331}, {pid=35, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure dcdab597c439e333702f5a0384e4db9a}] 2024-11-13T07:16:16,993 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 045378476123254a5c2616896d4edfcf 2024-11-13T07:16:16,996 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 1f49ca9e4320a113a985ca82c73f5c2a 2024-11-13T07:16:16,996 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 2e4b72fe1d04f499cf9c51ce52c79100 2024-11-13T07:16:16,997 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure dcdab597c439e333702f5a0384e4db9a 2024-11-13T07:16:16,997 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d029c66cacb99cb6bcc28a1f47b21331 2024-11-13T07:16:17,006 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 56b34ea82217cf90ce43d525dd7e1a9d 2024-11-13T07:16:17,007 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure dc0d3e1b8b1a5a44332ecc1f0118002b 2024-11-13T07:16:17,023 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9e8af980aa626886ac3cb035f55a2e4c 2024-11-13T07:16:17,033 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 1357138ab69f6578d72ce8d295f98b00 2024-11-13T07:16:17,033 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure ff8b864839731879b51af65fe2a46ac9 2024-11-13T07:16:26,400 ERROR [ProcedureDispatcherTimeoutThread {}] procedure2.RemoteProcedureDispatcher$TimeoutExecutorThread(331): DelayQueue for RemoteProcedureDispatcher is not empty when timed waiting elapsed. If this is repeated consistently, it means no element is getting expired from the queue and it might freeze the system. Queue: [containedObject=2998b885bdef,36543,1731482143970, timeout=1731482186998, delay=598, operations=[pid=28, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 045378476123254a5c2616896d4edfcf, pid=35, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure dcdab597c439e333702f5a0384e4db9a, pid=31, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9e8af980aa626886ac3cb035f55a2e4c], containedObject=2998b885bdef,39195,1731482143882, timeout=1731482186999, delay=599, operations=[pid=33, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 56b34ea82217cf90ce43d525dd7e1a9d, pid=29, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure ff8b864839731879b51af65fe2a46ac9, pid=27, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 1f49ca9e4320a113a985ca82c73f5c2a], containedObject=2998b885bdef,41365,1731482143808, timeout=1731482186999, delay=599, operations=[pid=30, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 1357138ab69f6578d72ce8d295f98b00, pid=34, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d029c66cacb99cb6bcc28a1f47b21331, pid=26, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 2e4b72fe1d04f499cf9c51ce52c79100, pid=32, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure dc0d3e1b8b1a5a44332ecc1f0118002b]] 2024-11-13T07:16:26,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=28 2024-11-13T07:16:27,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2024-11-13T07:16:27,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39195 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2024-11-13T07:16:27,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41365 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=30 2024-11-13T07:16:27,001 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=28}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on SPTestTable,33333332,1731482155666.045378476123254a5c2616896d4edfcf. 2024-11-13T07:16:27,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=31 2024-11-13T07:16:27,001 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 045378476123254a5c2616896d4edfcf: 2024-11-13T07:16:27,001 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on SPTestTable,e6666661,1731482155666.dcdab597c439e333702f5a0384e4db9a. 2024-11-13T07:16:27,001 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=28}] regionserver.SnapshotRegionCallable(75): Snapshotting region SPTestTable,33333332,1731482155666.045378476123254a5c2616896d4edfcf. for SnapshotProcedureTest completed. 2024-11-13T07:16:27,001 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=28}] snapshot.SnapshotManifest(241): Storing 'SPTestTable,33333332,1731482155666.045378476123254a5c2616896d4edfcf.' region-info for snapshot=SnapshotProcedureTest 2024-11-13T07:16:27,001 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing dcdab597c439e333702f5a0384e4db9a 1/1 column families, dataSize=435.70 KB heapSize=1.48 MB 2024-11-13T07:16:27,001 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=28}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-13T07:16:27,001 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=28}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-13T07:16:27,004 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d. 2024-11-13T07:16:27,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39195 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=29 2024-11-13T07:16:27,005 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2902): Flushing 56b34ea82217cf90ce43d525dd7e1a9d 1/1 column families, dataSize=21.42 KB heapSize=74.49 KB 2024-11-13T07:16:27,008 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=30}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on SPTestTable,66666664,1731482155666.1357138ab69f6578d72ce8d295f98b00. 2024-11-13T07:16:27,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41365 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=34 2024-11-13T07:16:27,008 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=30}] regionserver.HRegion(2603): Flush status journal for 1357138ab69f6578d72ce8d295f98b00: 2024-11-13T07:16:27,008 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=30}] regionserver.SnapshotRegionCallable(75): Snapshotting region SPTestTable,66666664,1731482155666.1357138ab69f6578d72ce8d295f98b00. for SnapshotProcedureTest completed. 2024-11-13T07:16:27,009 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=30}] snapshot.SnapshotManifest(241): Storing 'SPTestTable,66666664,1731482155666.1357138ab69f6578d72ce8d295f98b00.' region-info for snapshot=SnapshotProcedureTest 2024-11-13T07:16:27,009 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=30}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-13T07:16:27,009 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=30}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-13T07:16:27,012 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=34}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on SPTestTable,ccccccc8,1731482155666.d029c66cacb99cb6bcc28a1f47b21331. 2024-11-13T07:16:27,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41365 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=26 2024-11-13T07:16:27,013 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=34}] regionserver.HRegion(2902): Flushing d029c66cacb99cb6bcc28a1f47b21331 1/1 column families, dataSize=38.00 KB heapSize=131.98 KB 2024-11-13T07:16:27,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=39195 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=27 2024-11-13T07:16:27,016 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=29}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9. 2024-11-13T07:16:27,018 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=29}] regionserver.HRegion(2603): Flush status journal for ff8b864839731879b51af65fe2a46ac9: 2024-11-13T07:16:27,018 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=29}] regionserver.SnapshotRegionCallable(75): Snapshotting region SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9. for SnapshotProcedureTest completed. 2024-11-13T07:16:27,018 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=29}] snapshot.SnapshotManifest(241): Storing 'SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9.' region-info for snapshot=SnapshotProcedureTest 2024-11-13T07:16:27,018 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=29}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-13T07:16:27,018 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=29}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-13T07:16:27,024 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=31}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on SPTestTable,7ffffffd,1731482155666.9e8af980aa626886ac3cb035f55a2e4c. 2024-11-13T07:16:27,024 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=31}] regionserver.HRegion(2603): Flush status journal for 9e8af980aa626886ac3cb035f55a2e4c: 2024-11-13T07:16:27,025 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=31}] regionserver.SnapshotRegionCallable(75): Snapshotting region SPTestTable,7ffffffd,1731482155666.9e8af980aa626886ac3cb035f55a2e4c. for SnapshotProcedureTest completed. 2024-11-13T07:16:27,025 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=31}] snapshot.SnapshotManifest(241): Storing 'SPTestTable,7ffffffd,1731482155666.9e8af980aa626886ac3cb035f55a2e4c.' region-info for snapshot=SnapshotProcedureTest 2024-11-13T07:16:27,025 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=31}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-13T07:16:27,025 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=31}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-13T07:16:27,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41365 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=32 2024-11-13T07:16:27,029 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=26}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on SPTestTable,,1731482155666.2e4b72fe1d04f499cf9c51ce52c79100. 2024-11-13T07:16:27,029 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 2e4b72fe1d04f499cf9c51ce52c79100: 2024-11-13T07:16:27,029 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=26}] regionserver.SnapshotRegionCallable(75): Snapshotting region SPTestTable,,1731482155666.2e4b72fe1d04f499cf9c51ce52c79100. for SnapshotProcedureTest completed. 2024-11-13T07:16:27,029 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=26}] snapshot.SnapshotManifest(241): Storing 'SPTestTable,,1731482155666.2e4b72fe1d04f499cf9c51ce52c79100.' region-info for snapshot=SnapshotProcedureTest 2024-11-13T07:16:27,029 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=26}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-13T07:16:27,030 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=26}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-13T07:16:27,056 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=27}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a. 2024-11-13T07:16:27,057 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=27}] regionserver.HRegion(2603): Flush status journal for 1f49ca9e4320a113a985ca82c73f5c2a: 2024-11-13T07:16:27,057 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=27}] regionserver.SnapshotRegionCallable(75): Snapshotting region SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a. for SnapshotProcedureTest completed. 2024-11-13T07:16:27,057 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=27}] snapshot.SnapshotManifest(241): Storing 'SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a.' region-info for snapshot=SnapshotProcedureTest 2024-11-13T07:16:27,057 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=27}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-13T07:16:27,057 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=27}] snapshot.SnapshotManifest(256): Adding snapshot references for [] hfiles 2024-11-13T07:16:27,098 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/d029c66cacb99cb6bcc28a1f47b21331/.tmp/cf/cd4341f160924a8bbd597c5fa55da8b9 is 34, key is ccd/cf:cf/1731482175726/Put/seqid=0 2024-11-13T07:16:27,101 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/56b34ea82217cf90ce43d525dd7e1a9d/.tmp/cf/a5ea41c972c64a38a4d9030c966bf2bf is 34, key is baa/cf:cf/1731482175661/Put/seqid=0 2024-11-13T07:16:27,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741851_1027 (size=65) 2024-11-13T07:16:27,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741851_1027 (size=65) 2024-11-13T07:16:27,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741851_1027 (size=65) 2024-11-13T07:16:27,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741850_1026 (size=65) 2024-11-13T07:16:27,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741850_1026 (size=65) 2024-11-13T07:16:27,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741850_1026 (size=65) 2024-11-13T07:16:27,174 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/dcdab597c439e333702f5a0384e4db9a/.tmp/cf/45f93c04cf64493d9ad804767ff8bb3c is 34, key is eaa/cf:cf/1731482176556/Put/seqid=0 2024-11-13T07:16:27,178 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=31}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on SPTestTable,7ffffffd,1731482155666.9e8af980aa626886ac3cb035f55a2e4c. 2024-11-13T07:16:27,178 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=31}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=31 2024-11-13T07:16:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] master.HMaster(4169): Remote procedure done, pid=31 2024-11-13T07:16:27,179 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] procedure.SnapshotRegionProcedure(132): finish snapshot SnapshotProcedureTest on region 9e8af980aa626886ac3cb035f55a2e4c 2024-11-13T07:16:27,179 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 9e8af980aa626886ac3cb035f55a2e4c 2024-11-13T07:16:27,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=25, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 9e8af980aa626886ac3cb035f55a2e4c in 10.1960 sec 2024-11-13T07:16:27,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741849_1025 (size=65) 2024-11-13T07:16:27,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741849_1025 (size=65) 2024-11-13T07:16:27,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741849_1025 (size=65) 2024-11-13T07:16:27,189 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=28}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on SPTestTable,33333332,1731482155666.045378476123254a5c2616896d4edfcf. 2024-11-13T07:16:27,189 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-13T07:16:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-13T07:16:27,190 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] procedure.SnapshotRegionProcedure(132): finish snapshot SnapshotProcedureTest on region 045378476123254a5c2616896d4edfcf 2024-11-13T07:16:27,190 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 045378476123254a5c2616896d4edfcf 2024-11-13T07:16:27,193 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=25, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 045378476123254a5c2616896d4edfcf in 10.2070 sec 2024-11-13T07:16:27,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741854_1030 (size=65) 2024-11-13T07:16:27,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741854_1030 (size=65) 2024-11-13T07:16:27,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741854_1030 (size=65) 2024-11-13T07:16:27,232 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=27}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a. 2024-11-13T07:16:27,232 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=27 2024-11-13T07:16:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] master.HMaster(4169): Remote procedure done, pid=27 2024-11-13T07:16:27,233 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] procedure.SnapshotRegionProcedure(132): finish snapshot SnapshotProcedureTest on region 1f49ca9e4320a113a985ca82c73f5c2a 2024-11-13T07:16:27,233 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 1f49ca9e4320a113a985ca82c73f5c2a 2024-11-13T07:16:27,235 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=25, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 1f49ca9e4320a113a985ca82c73f5c2a in 10.2500 sec 2024-11-13T07:16:27,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741853_1029 (size=65) 2024-11-13T07:16:27,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741853_1029 (size=65) 2024-11-13T07:16:27,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741853_1029 (size=65) 2024-11-13T07:16:27,282 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=29}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9. 2024-11-13T07:16:27,282 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=29 2024-11-13T07:16:27,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741852_1028 (size=57) 2024-11-13T07:16:27,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741852_1028 (size=57) 2024-11-13T07:16:27,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] master.HMaster(4169): Remote procedure done, pid=29 2024-11-13T07:16:27,283 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] procedure.SnapshotRegionProcedure(132): finish snapshot SnapshotProcedureTest on region ff8b864839731879b51af65fe2a46ac9 2024-11-13T07:16:27,283 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure ff8b864839731879b51af65fe2a46ac9 2024-11-13T07:16:27,284 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=26}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on SPTestTable,,1731482155666.2e4b72fe1d04f499cf9c51ce52c79100. 2024-11-13T07:16:27,284 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-13T07:16:27,285 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=32}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b. 2024-11-13T07:16:27,285 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=32}] regionserver.HRegion(2902): Flushing dc0d3e1b8b1a5a44332ecc1f0118002b 1/1 column families, dataSize=19.80 KB heapSize=68.91 KB 2024-11-13T07:16:27,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-13T07:16:27,286 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] procedure.SnapshotRegionProcedure(132): finish snapshot SnapshotProcedureTest on region 2e4b72fe1d04f499cf9c51ce52c79100 2024-11-13T07:16:27,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741852_1028 (size=57) 2024-11-13T07:16:27,287 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 2e4b72fe1d04f499cf9c51ce52c79100 2024-11-13T07:16:27,288 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=25, state=SUCCESS, hasLock=false; SnapshotRegionProcedure ff8b864839731879b51af65fe2a46ac9 in 10.3010 sec 2024-11-13T07:16:27,292 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 2e4b72fe1d04f499cf9c51ce52c79100 in 10.3050 sec 2024-11-13T07:16:27,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741856_1032 (size=499309) 2024-11-13T07:16:27,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741856_1032 (size=499309) 2024-11-13T07:16:27,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741856_1032 (size=499309) 2024-11-13T07:16:27,361 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=435.70 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/dcdab597c439e333702f5a0384e4db9a/.tmp/cf/45f93c04cf64493d9ad804767ff8bb3c 2024-11-13T07:16:27,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741855_1031 (size=28623) 2024-11-13T07:16:27,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741855_1031 (size=28623) 2024-11-13T07:16:27,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741855_1031 (size=28623) 2024-11-13T07:16:27,380 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=21.42 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/56b34ea82217cf90ce43d525dd7e1a9d/.tmp/cf/a5ea41c972c64a38a4d9030c966bf2bf 2024-11-13T07:16:27,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741857_1033 (size=47197) 2024-11-13T07:16:27,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741857_1033 (size=47197) 2024-11-13T07:16:27,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741857_1033 (size=47197) 2024-11-13T07:16:27,393 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.00 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/d029c66cacb99cb6bcc28a1f47b21331/.tmp/cf/cd4341f160924a8bbd597c5fa55da8b9 2024-11-13T07:16:27,417 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/56b34ea82217cf90ce43d525dd7e1a9d/.tmp/cf/a5ea41c972c64a38a4d9030c966bf2bf as hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/56b34ea82217cf90ce43d525dd7e1a9d/cf/a5ea41c972c64a38a4d9030c966bf2bf 2024-11-13T07:16:27,433 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/56b34ea82217cf90ce43d525dd7e1a9d/cf/a5ea41c972c64a38a4d9030c966bf2bf, entries=731, sequenceid=5, filesize=28.0 K 2024-11-13T07:16:27,434 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3140): Finished flush of dataSize ~21.42 KB/21930, heapSize ~74.48 KB/76264, currentSize=0 B/0 for 56b34ea82217cf90ce43d525dd7e1a9d in 430ms, sequenceid=5, compaction requested=false 2024-11-13T07:16:27,435 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2603): Flush status journal for 56b34ea82217cf90ce43d525dd7e1a9d: 2024-11-13T07:16:27,435 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d. for SnapshotProcedureTest completed. 2024-11-13T07:16:27,435 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(241): Storing 'SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d.' region-info for snapshot=SnapshotProcedureTest 2024-11-13T07:16:27,435 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-13T07:16:27,435 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/56b34ea82217cf90ce43d525dd7e1a9d/cf/a5ea41c972c64a38a4d9030c966bf2bf] hfiles 2024-11-13T07:16:27,435 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/56b34ea82217cf90ce43d525dd7e1a9d/cf/a5ea41c972c64a38a4d9030c966bf2bf for snapshot=SnapshotProcedureTest 2024-11-13T07:16:27,444 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/dcdab597c439e333702f5a0384e4db9a/.tmp/cf/45f93c04cf64493d9ad804767ff8bb3c as hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/dcdab597c439e333702f5a0384e4db9a/cf/45f93c04cf64493d9ad804767ff8bb3c 2024-11-13T07:16:27,455 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/dc0d3e1b8b1a5a44332ecc1f0118002b/.tmp/cf/02c8b19f5da7401b8ca21a18ca0c0561 is 34, key is aaa/cf:cf/1731482175640/Put/seqid=0 2024-11-13T07:16:27,457 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/dcdab597c439e333702f5a0384e4db9a/cf/45f93c04cf64493d9ad804767ff8bb3c, entries=14872, sequenceid=5, filesize=487.6 K 2024-11-13T07:16:27,459 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~435.70 KB/446160, heapSize ~1.48 MB/1546928, currentSize=0 B/0 for dcdab597c439e333702f5a0384e4db9a in 457ms, sequenceid=5, compaction requested=false 2024-11-13T07:16:27,459 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for dcdab597c439e333702f5a0384e4db9a: 2024-11-13T07:16:27,459 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region SPTestTable,e6666661,1731482155666.dcdab597c439e333702f5a0384e4db9a. for SnapshotProcedureTest completed. 2024-11-13T07:16:27,459 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(241): Storing 'SPTestTable,e6666661,1731482155666.dcdab597c439e333702f5a0384e4db9a.' region-info for snapshot=SnapshotProcedureTest 2024-11-13T07:16:27,459 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-13T07:16:27,459 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/dcdab597c439e333702f5a0384e4db9a/cf/45f93c04cf64493d9ad804767ff8bb3c] hfiles 2024-11-13T07:16:27,459 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/dcdab597c439e333702f5a0384e4db9a/cf/45f93c04cf64493d9ad804767ff8bb3c for snapshot=SnapshotProcedureTest 2024-11-13T07:16:27,469 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/d029c66cacb99cb6bcc28a1f47b21331/.tmp/cf/cd4341f160924a8bbd597c5fa55da8b9 as hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/d029c66cacb99cb6bcc28a1f47b21331/cf/cd4341f160924a8bbd597c5fa55da8b9 2024-11-13T07:16:27,481 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/d029c66cacb99cb6bcc28a1f47b21331/cf/cd4341f160924a8bbd597c5fa55da8b9, entries=1297, sequenceid=5, filesize=46.1 K 2024-11-13T07:16:27,482 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=34}] regionserver.HRegion(3140): Finished flush of dataSize ~38.00 KB/38910, heapSize ~131.96 KB/135128, currentSize=0 B/0 for d029c66cacb99cb6bcc28a1f47b21331 in 470ms, sequenceid=5, compaction requested=false 2024-11-13T07:16:27,482 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=34}] regionserver.HRegion(2603): Flush status journal for d029c66cacb99cb6bcc28a1f47b21331: 2024-11-13T07:16:27,482 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=34}] regionserver.SnapshotRegionCallable(75): Snapshotting region SPTestTable,ccccccc8,1731482155666.d029c66cacb99cb6bcc28a1f47b21331. for SnapshotProcedureTest completed. 2024-11-13T07:16:27,483 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=34}] snapshot.SnapshotManifest(241): Storing 'SPTestTable,ccccccc8,1731482155666.d029c66cacb99cb6bcc28a1f47b21331.' region-info for snapshot=SnapshotProcedureTest 2024-11-13T07:16:27,483 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=34}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-13T07:16:27,483 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=34}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/d029c66cacb99cb6bcc28a1f47b21331/cf/cd4341f160924a8bbd597c5fa55da8b9] hfiles 2024-11-13T07:16:27,483 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=34}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/d029c66cacb99cb6bcc28a1f47b21331/cf/cd4341f160924a8bbd597c5fa55da8b9 for snapshot=SnapshotProcedureTest 2024-11-13T07:16:27,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741858_1034 (size=105) 2024-11-13T07:16:27,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741858_1034 (size=105) 2024-11-13T07:16:27,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741858_1034 (size=105) 2024-11-13T07:16:27,524 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d. 2024-11-13T07:16:27,524 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-11-13T07:16:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-11-13T07:16:27,526 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] procedure.SnapshotRegionProcedure(132): finish snapshot SnapshotProcedureTest on region 56b34ea82217cf90ce43d525dd7e1a9d 2024-11-13T07:16:27,526 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 56b34ea82217cf90ce43d525dd7e1a9d 2024-11-13T07:16:27,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=25, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 56b34ea82217cf90ce43d525dd7e1a9d in 10.5450 sec 2024-11-13T07:16:27,540 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=30}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on SPTestTable,66666664,1731482155666.1357138ab69f6578d72ce8d295f98b00. 2024-11-13T07:16:27,540 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-11-13T07:16:27,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] master.HMaster(4169): Remote procedure done, pid=30 2024-11-13T07:16:27,541 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] procedure.SnapshotRegionProcedure(132): finish snapshot SnapshotProcedureTest on region 1357138ab69f6578d72ce8d295f98b00 2024-11-13T07:16:27,541 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure 1357138ab69f6578d72ce8d295f98b00 2024-11-13T07:16:27,543 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=25, state=SUCCESS, hasLock=false; SnapshotRegionProcedure 1357138ab69f6578d72ce8d295f98b00 in 10.5580 sec 2024-11-13T07:16:27,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741859_1035 (size=26918) 2024-11-13T07:16:27,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741859_1035 (size=26918) 2024-11-13T07:16:27,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741859_1035 (size=26918) 2024-11-13T07:16:27,555 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=19.80 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/dc0d3e1b8b1a5a44332ecc1f0118002b/.tmp/cf/02c8b19f5da7401b8ca21a18ca0c0561 2024-11-13T07:16:27,565 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/dc0d3e1b8b1a5a44332ecc1f0118002b/.tmp/cf/02c8b19f5da7401b8ca21a18ca0c0561 as hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/dc0d3e1b8b1a5a44332ecc1f0118002b/cf/02c8b19f5da7401b8ca21a18ca0c0561 2024-11-13T07:16:27,575 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/dc0d3e1b8b1a5a44332ecc1f0118002b/cf/02c8b19f5da7401b8ca21a18ca0c0561, entries=676, sequenceid=5, filesize=26.3 K 2024-11-13T07:16:27,577 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~19.80 KB/20280, heapSize ~68.89 KB/70544, currentSize=0 B/0 for dc0d3e1b8b1a5a44332ecc1f0118002b in 291ms, sequenceid=5, compaction requested=false 2024-11-13T07:16:27,577 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=32}] regionserver.HRegion(2603): Flush status journal for dc0d3e1b8b1a5a44332ecc1f0118002b: 2024-11-13T07:16:27,577 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=32}] regionserver.SnapshotRegionCallable(75): Snapshotting region SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b. for SnapshotProcedureTest completed. 2024-11-13T07:16:27,577 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=32}] snapshot.SnapshotManifest(241): Storing 'SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b.' region-info for snapshot=SnapshotProcedureTest 2024-11-13T07:16:27,577 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=32}] snapshot.SnapshotManifest(246): Creating references for hfiles 2024-11-13T07:16:27,577 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=32}] snapshot.SnapshotManifest(256): Adding snapshot references for [hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/dc0d3e1b8b1a5a44332ecc1f0118002b/cf/02c8b19f5da7401b8ca21a18ca0c0561] hfiles 2024-11-13T07:16:27,577 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=32}] snapshot.SnapshotManifest(265): Adding reference for file (1/1): hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/dc0d3e1b8b1a5a44332ecc1f0118002b/cf/02c8b19f5da7401b8ca21a18ca0c0561 for snapshot=SnapshotProcedureTest 2024-11-13T07:16:27,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741861_1037 (size=105) 2024-11-13T07:16:27,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741861_1037 (size=105) 2024-11-13T07:16:27,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741861_1037 (size=105) 2024-11-13T07:16:27,584 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=34}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on SPTestTable,ccccccc8,1731482155666.d029c66cacb99cb6bcc28a1f47b21331. 2024-11-13T07:16:27,584 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=34}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=34 2024-11-13T07:16:27,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] master.HMaster(4169): Remote procedure done, pid=34 2024-11-13T07:16:27,584 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] procedure.SnapshotRegionProcedure(132): finish snapshot SnapshotProcedureTest on region d029c66cacb99cb6bcc28a1f47b21331 2024-11-13T07:16:27,585 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=34, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure d029c66cacb99cb6bcc28a1f47b21331 2024-11-13T07:16:27,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741860_1036 (size=97) 2024-11-13T07:16:27,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741860_1036 (size=97) 2024-11-13T07:16:27,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741860_1036 (size=97) 2024-11-13T07:16:27,588 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=25, state=SUCCESS, hasLock=false; SnapshotRegionProcedure d029c66cacb99cb6bcc28a1f47b21331 in 10.6020 sec 2024-11-13T07:16:27,588 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on SPTestTable,e6666661,1731482155666.dcdab597c439e333702f5a0384e4db9a. 2024-11-13T07:16:27,588 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-13T07:16:27,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-13T07:16:27,589 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] procedure.SnapshotRegionProcedure(132): finish snapshot SnapshotProcedureTest on region dcdab597c439e333702f5a0384e4db9a 2024-11-13T07:16:27,589 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure dcdab597c439e333702f5a0384e4db9a 2024-11-13T07:16:27,591 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=25, state=SUCCESS, hasLock=false; SnapshotRegionProcedure dcdab597c439e333702f5a0384e4db9a in 10.6060 sec 2024-11-13T07:16:27,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741862_1038 (size=105) 2024-11-13T07:16:27,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741862_1038 (size=105) 2024-11-13T07:16:27,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741862_1038 (size=105) 2024-11-13T07:16:27,602 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=32}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b. 2024-11-13T07:16:27,602 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=32 2024-11-13T07:16:27,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] master.HMaster(4169): Remote procedure done, pid=32 2024-11-13T07:16:27,603 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] procedure.SnapshotRegionProcedure(132): finish snapshot SnapshotProcedureTest on region dc0d3e1b8b1a5a44332ecc1f0118002b 2024-11-13T07:16:27,603 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=25, state=RUNNABLE, hasLock=false; SnapshotRegionProcedure dc0d3e1b8b1a5a44332ecc1f0118002b 2024-11-13T07:16:27,606 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=25 2024-11-13T07:16:27,606 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=25, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=25, snapshot={ ss=SnapshotProcedureTest table=SPTestTable type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2024-11-13T07:16:27,606 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=25, state=SUCCESS, hasLock=false; SnapshotRegionProcedure dc0d3e1b8b1a5a44332ecc1f0118002b in 10.6200 sec 2024-11-13T07:16:27,607 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=25, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=25, snapshot={ ss=SnapshotProcedureTest table=SPTestTable type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2024-11-13T07:16:27,607 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=25, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=25, snapshot={ ss=SnapshotProcedureTest table=SPTestTable type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2024-11-13T07:16:27,608 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(495): Convert to Single Snapshot Manifest for SnapshotProcedureTest 2024-11-13T07:16:27,609 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(130): No regions under directory:hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/.hbase-snapshot/.tmp/SnapshotProcedureTest 2024-11-13T07:16:27,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741863_1039 (size=1174) 2024-11-13T07:16:27,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741863_1039 (size=1174) 2024-11-13T07:16:27,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741863_1039 (size=1174) 2024-11-13T07:16:27,640 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=25, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=25, snapshot={ ss=SnapshotProcedureTest table=SPTestTable type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2024-11-13T07:16:27,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=25, state=RUNNABLE, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest}, {pid=37, ppid=25, state=RUNNABLE, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest}, {pid=38, ppid=25, state=RUNNABLE, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest}, {pid=39, ppid=25, state=RUNNABLE, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest}, {pid=40, ppid=25, state=RUNNABLE, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest}, {pid=41, ppid=25, state=RUNNABLE, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest}, {pid=42, ppid=25, state=RUNNABLE, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest}, {pid=43, ppid=25, state=RUNNABLE, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest}, {pid=44, ppid=25, state=RUNNABLE, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest}, {pid=45, ppid=25, state=RUNNABLE, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest}] 2024-11-13T07:16:27,649 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1464): pid=45, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest Acquired verify snapshot worker=2998b885bdef,39195,1731482143882 2024-11-13T07:16:27,649 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1464): pid=37, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest Acquired verify snapshot worker=2998b885bdef,36543,1731482143970 2024-11-13T07:16:27,649 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1464): pid=39, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest Acquired verify snapshot worker=2998b885bdef,39195,1731482143882 2024-11-13T07:16:27,649 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1464): pid=38, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest Acquired verify snapshot worker=2998b885bdef,36543,1731482143970 2024-11-13T07:16:27,649 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1464): pid=36, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest Acquired verify snapshot worker=2998b885bdef,39195,1731482143882 2024-11-13T07:16:27,655 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1464): pid=44, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest Acquired verify snapshot worker=2998b885bdef,36543,1731482143970 2024-11-13T07:16:27,656 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1464): pid=42, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest Acquired verify snapshot worker=2998b885bdef,41365,1731482143808 2024-11-13T07:16:27,656 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1464): pid=43, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest Acquired verify snapshot worker=2998b885bdef,41365,1731482143808 2024-11-13T07:16:27,657 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1464): pid=41, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest Acquired verify snapshot worker=2998b885bdef,41365,1731482143808 2024-11-13T07:16:27,725 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T07:16:27,726 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 2998b885bdef,39195,1731482143882 2024-11-13T07:16:27,726 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 2998b885bdef,39195,1731482143882: Simulated kill ***** 2024-11-13T07:16:27,726 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-13T07:16:27,726 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-13T07:16:27,727 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-13T07:16:27,728 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-13T07:16:27,728 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-13T07:16:27,728 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 496551008 }, "NonHeapMemoryUsage": { "committed": 177471488, "init": 7667712, "max": -1, "used": 173272496 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [], "beans": [], "beans": [] } 2024-11-13T07:16:27,729 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44607 {}] master.MasterRpcServices(700): 2998b885bdef,39195,1731482143882 reported a fatal error: ***** ABORTING region server 2998b885bdef,39195,1731482143882: Simulated kill ***** 2024-11-13T07:16:27,729 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2998b885bdef,39195,1731482143882' ***** 2024-11-13T07:16:27,729 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-13T07:16:27,730 INFO [RS:1;2998b885bdef:39195 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T07:16:27,730 INFO [RS:1;2998b885bdef:39195 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-13T07:16:27,730 INFO [RS:1;2998b885bdef:39195 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-13T07:16:27,730 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T07:16:27,730 INFO [RS:1;2998b885bdef:39195 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T07:16:27,730 INFO [RS:1;2998b885bdef:39195 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;2998b885bdef:39195. 2024-11-13T07:16:27,730 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T07:16:27,730 DEBUG [RS:1;2998b885bdef:39195 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T07:16:27,730 DEBUG [RS:1;2998b885bdef:39195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:16:27,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741833_1009 (size=1488) 2024-11-13T07:16:27,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741833_1009 (size=1488) 2024-11-13T07:16:27,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741833_1009 (size=1488) 2024-11-13T07:16:27,735 DEBUG [RS:1;2998b885bdef:39195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:16:27,735 INFO [RS:1;2998b885bdef:39195 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T07:16:27,736 INFO [RS:1;2998b885bdef:39195 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39195 2024-11-13T07:16:27,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T07:16:27,747 INFO [RS:1;2998b885bdef:39195 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T07:16:27,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39195-0x101330cc6ab0002, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2998b885bdef,39195,1731482143882 2024-11-13T07:16:27,756 INFO [regionserver/2998b885bdef:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T07:16:27,760 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2998b885bdef,39195,1731482143882] 2024-11-13T07:16:27,778 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2998b885bdef,39195,1731482143882 already deleted, retry=false 2024-11-13T07:16:27,778 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 2998b885bdef,39195,1731482143882 on 2998b885bdef,44607,1731482143517 2024-11-13T07:16:27,780 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 2998b885bdef,39195,1731482143882, splitWal=true, meta=false 2024-11-13T07:16:27,780 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=46 for 2998b885bdef,39195,1731482143882 (carryingMeta=false) 2998b885bdef,39195,1731482143882/CRASHED/regionCount=3/lock=java.util.concurrent.locks.ReentrantReadWriteLock@54a7857e[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-13T07:16:27,783 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(169): Start pid=46, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 2998b885bdef,39195,1731482143882, splitWal=true, meta=false 2024-11-13T07:16:27,785 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(207): 2998b885bdef,39195,1731482143882 had 3 regions 2024-11-13T07:16:27,787 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=46, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 2998b885bdef,39195,1731482143882, splitWal=true, meta=false, isMeta: false 2024-11-13T07:16:27,788 DEBUG [PEWorker-5 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,39195,1731482143882-splitting 2024-11-13T07:16:27,790 INFO [PEWorker-5 {}] master.SplitWALManager(105): 2998b885bdef,39195,1731482143882 WAL count=1, meta=false 2024-11-13T07:16:27,790 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 2998b885bdef%2C39195%2C1731482143882.1731482145268}] 2024-11-13T07:16:27,793 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=2998b885bdef,36543,1731482143970 2024-11-13T07:16:27,793 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1473): pid=45, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,39195,1731482143882 Release verify snapshot worker=2998b885bdef,39195,1731482143882 2024-11-13T07:16:27,793 WARN [PEWorker-4 {}] procedure.ServerRemoteProcedure(104): Can not send remote operation 45 to 2998b885bdef,39195,1731482143882, this operation will be retried to send to another server 2024-11-13T07:16:27,793 WARN [PEWorker-4 {}] procedure.SnapshotVerifyProcedure(169): Failed to get snapshot verify result , wait 1000 ms to retry org.apache.hadoop.hbase.procedure2.FailedRemoteDispatchException: Failed sent request at org.apache.hadoop.hbase.master.procedure.SnapshotVerifyProcedure.execute(SnapshotVerifyProcedure.java:156) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotVerifyProcedure.execute(SnapshotVerifyProcedure.java:52) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T07:16:27,793 INFO [PEWorker-4 {}] procedure2.TimeoutExecutorThread(82): ADDED pid=45, ppid=25, state=WAITING_TIMEOUT, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest; timeout=1000, timestamp=1731482188793 2024-11-13T07:16:27,795 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 2998b885bdef%2C39195%2C1731482143882.1731482145268, worker=2998b885bdef,36543,1731482143970}] 2024-11-13T07:16:27,797 INFO [RegionServerTracker-0 {}] procedure.SnapshotRegionProcedure(123): This procedure 33 is already finished, skip the rest processes 2024-11-13T07:16:27,797 INFO [RegionServerTracker-0 {}] procedure.SnapshotRegionProcedure(123): This procedure 29 is already finished, skip the rest processes 2024-11-13T07:16:27,797 INFO [RegionServerTracker-0 {}] procedure.SnapshotRegionProcedure(123): This procedure 27 is already finished, skip the rest processes 2024-11-13T07:16:27,797 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1473): pid=39, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,39195,1731482143882 Release verify snapshot worker=2998b885bdef,39195,1731482143882 2024-11-13T07:16:27,797 WARN [PEWorker-1 {}] procedure.ServerRemoteProcedure(104): Can not send remote operation 39 to 2998b885bdef,39195,1731482143882, this operation will be retried to send to another server 2024-11-13T07:16:27,797 WARN [PEWorker-1 {}] procedure.SnapshotVerifyProcedure(169): Failed to get snapshot verify result , wait 1000 ms to retry org.apache.hadoop.hbase.procedure2.FailedRemoteDispatchException: Failed sent request at org.apache.hadoop.hbase.master.procedure.SnapshotVerifyProcedure.execute(SnapshotVerifyProcedure.java:156) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotVerifyProcedure.execute(SnapshotVerifyProcedure.java:52) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T07:16:27,797 INFO [PEWorker-1 {}] procedure2.TimeoutExecutorThread(82): ADDED pid=39, ppid=25, state=WAITING_TIMEOUT, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest; timeout=1000, timestamp=1731482188797 2024-11-13T07:16:27,802 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1473): pid=36, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,39195,1731482143882 Release verify snapshot worker=2998b885bdef,39195,1731482143882 2024-11-13T07:16:27,802 WARN [PEWorker-5 {}] procedure.ServerRemoteProcedure(104): Can not send remote operation 36 to 2998b885bdef,39195,1731482143882, this operation will be retried to send to another server 2024-11-13T07:16:27,802 WARN [PEWorker-5 {}] procedure.SnapshotVerifyProcedure(169): Failed to get snapshot verify result , wait 1000 ms to retry org.apache.hadoop.hbase.procedure2.FailedRemoteDispatchException: Failed sent request at org.apache.hadoop.hbase.master.procedure.SnapshotVerifyProcedure.execute(SnapshotVerifyProcedure.java:156) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotVerifyProcedure.execute(SnapshotVerifyProcedure.java:52) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T07:16:27,802 INFO [PEWorker-5 {}] procedure2.TimeoutExecutorThread(82): ADDED pid=36, ppid=25, state=WAITING_TIMEOUT, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest; timeout=1000, timestamp=1731482188802 2024-11-13T07:16:27,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39195-0x101330cc6ab0002, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T07:16:27,861 INFO [RS:1;2998b885bdef:39195 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T07:16:27,861 INFO [RS:1;2998b885bdef:39195 {}] regionserver.HRegionServer(1031): Exiting; stopping=2998b885bdef,39195,1731482143882; zookeeper connection closed. 2024-11-13T07:16:27,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39195-0x101330cc6ab0002, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T07:16:27,861 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7940ff32 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7940ff32 2024-11-13T07:16:28,124 INFO [regionserver/2998b885bdef:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T07:16:28,125 INFO [regionserver/2998b885bdef:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T07:16:35,628 INFO [master/2998b885bdef:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-13T07:16:35,628 INFO [master/2998b885bdef:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-13T07:16:37,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotVerifyCallable, pid=37 2024-11-13T07:16:37,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotVerifyCallable, pid=38 2024-11-13T07:16:37,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=48 2024-11-13T07:16:37,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotVerifyCallable, pid=44 2024-11-13T07:16:37,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41365 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotVerifyCallable, pid=42 2024-11-13T07:16:37,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41365 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotVerifyCallable, pid=43 2024-11-13T07:16:37,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41365 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotVerifyCallable, pid=41 2024-11-13T07:16:37,686 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_VERIFY_SNAPSHOT, pid=42}] regionserver.RSSnapshotVerifier$SnapshotManifestCacheLoader(144): loading snapshot manifest for SnapshotProcedureTest from hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/.hbase-snapshot/.tmp/SnapshotProcedureTest 2024-11-13T07:16:37,687 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_VERIFY_SNAPSHOT, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-13T07:16:37,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-13T07:16:37,690 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_VERIFY_SNAPSHOT, pid=37}] regionserver.RSSnapshotVerifier$SnapshotManifestCacheLoader(144): loading snapshot manifest for SnapshotProcedureTest from hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/.hbase-snapshot/.tmp/SnapshotProcedureTest 2024-11-13T07:16:37,690 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_VERIFY_SNAPSHOT, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-13T07:16:37,690 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_VERIFY_SNAPSHOT, pid=38}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=38 2024-11-13T07:16:37,690 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_VERIFY_SNAPSHOT, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-13T07:16:37,690 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_VERIFY_SNAPSHOT, pid=42}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=42 2024-11-13T07:16:37,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44607 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-13T07:16:37,696 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=48}] wal.WALSplitter(299): Splitting hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,39195,1731482143882-splitting/2998b885bdef%2C39195%2C1731482143882.1731482145268, size=1.5 K (1488bytes) 2024-11-13T07:16:37,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44607 {}] master.HMaster(4169): Remote procedure done, pid=38 2024-11-13T07:16:37,696 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=48}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,39195,1731482143882-splitting/2998b885bdef%2C39195%2C1731482143882.1731482145268 2024-11-13T07:16:37,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-13T07:16:37,698 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=48}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,39195,1731482143882-splitting/2998b885bdef%2C39195%2C1731482143882.1731482145268 after 2ms 2024-11-13T07:16:37,698 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1473): pid=41, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,41365,1731482143808 Release verify snapshot worker=2998b885bdef,41365,1731482143808 2024-11-13T07:16:37,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] master.HMaster(4169): Remote procedure done, pid=42 2024-11-13T07:16:37,700 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 {event_type=RS_VERIFY_SNAPSHOT, pid=44}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=44 2024-11-13T07:16:37,702 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1473): pid=37, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,36543,1731482143970 Release verify snapshot worker=2998b885bdef,36543,1731482143970 2024-11-13T07:16:37,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] master.HMaster(4169): Remote procedure done, pid=44 2024-11-13T07:16:37,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=25, state=SUCCESS, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,41365,1731482143808 in 10.0530 sec 2024-11-13T07:16:37,709 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=48}] wal.WALSplitter(310): Open hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,39195,1731482143882-splitting/2998b885bdef%2C39195%2C1731482143882.1731482145268 took 12ms 2024-11-13T07:16:37,710 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=25, state=SUCCESS, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,36543,1731482143970 in 10.0570 sec 2024-11-13T07:16:37,713 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1473): pid=43, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,41365,1731482143808 Release verify snapshot worker=2998b885bdef,41365,1731482143808 2024-11-13T07:16:37,714 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1464): pid=45, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest Acquired verify snapshot worker=2998b885bdef,36543,1731482143970 2024-11-13T07:16:37,716 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1473): pid=44, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,36543,1731482143970 Release verify snapshot worker=2998b885bdef,36543,1731482143970 2024-11-13T07:16:37,716 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1473): pid=42, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,41365,1731482143808 Release verify snapshot worker=2998b885bdef,41365,1731482143808 2024-11-13T07:16:37,719 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1464): pid=40, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest Acquired verify snapshot worker=2998b885bdef,36543,1731482143970 2024-11-13T07:16:37,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=25, state=SUCCESS, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,36543,1731482143970 in 10.0710 sec 2024-11-13T07:16:37,724 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=48}] wal.WALSplitter(352): Last flushed sequenceid for 1f49ca9e4320a113a985ca82c73f5c2a: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "cf" sequence_id: 18446744073709551615 } 2024-11-13T07:16:37,727 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=48}] wal.WALSplitter(352): Last flushed sequenceid for 56b34ea82217cf90ce43d525dd7e1a9d: last_flushed_sequence_id: 5 store_sequence_id { family_name: "cf" sequence_id: 5 } 2024-11-13T07:16:37,728 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1473): pid=38, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,36543,1731482143970 Release verify snapshot worker=2998b885bdef,36543,1731482143970 2024-11-13T07:16:37,729 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=25, state=SUCCESS, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,41365,1731482143808 in 10.0680 sec 2024-11-13T07:16:37,741 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1464): pid=36, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest Acquired verify snapshot worker=2998b885bdef,36543,1731482143970 2024-11-13T07:16:37,741 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1464): pid=39, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest Acquired verify snapshot worker=2998b885bdef,41365,1731482143808 2024-11-13T07:16:37,741 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=25, state=SUCCESS, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,36543,1731482143970 in 10.0830 sec 2024-11-13T07:16:37,752 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=48}] wal.WALSplitter(352): Last flushed sequenceid for ff8b864839731879b51af65fe2a46ac9: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "cf" sequence_id: 18446744073709551615 } 2024-11-13T07:16:37,752 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=48}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,39195,1731482143882-splitting/2998b885bdef%2C39195%2C1731482143882.1731482145268 so closing down 2024-11-13T07:16:37,752 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=48}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T07:16:37,753 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=48}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T07:16:37,753 INFO [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=48}] wal.WALSplitter(425): Processed 5 edits across 0 Regions in 38 ms; skipped=5; WAL=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,39195,1731482143882-splitting/2998b885bdef%2C39195%2C1731482143882.1731482145268, size=1.5 K, length=1488, corrupted=false, cancelled=false 2024-11-13T07:16:37,753 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=48}] wal.WALSplitter(428): Completed split of hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,39195,1731482143882-splitting/2998b885bdef%2C39195%2C1731482143882.1731482145268, journal: Splitting hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,39195,1731482143882-splitting/2998b885bdef%2C39195%2C1731482143882.1731482145268, size=1.5 K (1488bytes) at 1731482197696Finishing writing output for hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,39195,1731482143882-splitting/2998b885bdef%2C39195%2C1731482143882.1731482145268 so closing down at 1731482197752 (+56 ms)3 split writer threads finished at 1731482197753 (+1 ms)Processed 5 edits across 0 Regions in 38 ms; skipped=5; WAL=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,39195,1731482143882-splitting/2998b885bdef%2C39195%2C1731482143882.1731482145268, size=1.5 K, length=1488, corrupted=false, cancelled=false at 1731482197753 2024-11-13T07:16:37,753 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=48}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,39195,1731482143882-splitting/2998b885bdef%2C39195%2C1731482143882.1731482145268 2024-11-13T07:16:37,754 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2998b885bdef:0-0 {event_type=RS_LOG_REPLAY, pid=48}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=48 2024-11-13T07:16:37,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] master.HMaster(4169): Remote procedure done, pid=48 2024-11-13T07:16:37,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=25, state=SUCCESS, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,41365,1731482143808 in 10.0710 sec 2024-11-13T07:16:37,774 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,39195,1731482143882-splitting/2998b885bdef%2C39195%2C1731482143882.1731482145268 to hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/oldWALs 2024-11-13T07:16:37,777 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=47 2024-11-13T07:16:37,777 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=47, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2998b885bdef%2C39195%2C1731482143882.1731482145268, worker=2998b885bdef,36543,1731482143970 in 9.9790 sec 2024-11-13T07:16:37,778 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=2998b885bdef,36543,1731482143970 2024-11-13T07:16:37,783 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-13T07:16:37,783 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; SplitWALProcedure 2998b885bdef%2C39195%2C1731482143882.1731482145268, worker=2998b885bdef,36543,1731482143970 in 9.9880 sec 2024-11-13T07:16:37,785 INFO [PEWorker-5 {}] master.SplitLogManager(171): hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,39195,1731482143882-splitting dir is empty, no logs to split. 2024-11-13T07:16:37,785 INFO [PEWorker-5 {}] master.SplitWALManager(105): 2998b885bdef,39195,1731482143882 WAL count=0, meta=false 2024-11-13T07:16:37,785 DEBUG [PEWorker-5 {}] procedure.ServerCrashProcedure(329): Check if 2998b885bdef,39195,1731482143882 WAL splitting is done? wals=0, meta=false 2024-11-13T07:16:37,788 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 2998b885bdef,39195,1731482143882 failed, ignore...File hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,39195,1731482143882-splitting does not exist. 2024-11-13T07:16:37,789 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=46, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=ff8b864839731879b51af65fe2a46ac9, ASSIGN}, {pid=50, ppid=46, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=1f49ca9e4320a113a985ca82c73f5c2a, ASSIGN}, {pid=51, ppid=46, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=56b34ea82217cf90ce43d525dd7e1a9d, ASSIGN}] 2024-11-13T07:16:37,791 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=49, ppid=46, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=ff8b864839731879b51af65fe2a46ac9, ASSIGN 2024-11-13T07:16:37,791 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=51, ppid=46, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=56b34ea82217cf90ce43d525dd7e1a9d, ASSIGN 2024-11-13T07:16:37,792 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=50, ppid=46, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=1f49ca9e4320a113a985ca82c73f5c2a, ASSIGN 2024-11-13T07:16:37,793 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=49, ppid=46, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=ff8b864839731879b51af65fe2a46ac9, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T07:16:37,793 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=51, ppid=46, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=56b34ea82217cf90ce43d525dd7e1a9d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T07:16:37,793 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=50, ppid=46, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=1f49ca9e4320a113a985ca82c73f5c2a, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T07:16:37,943 DEBUG [2998b885bdef:44607 {}] balancer.BalancerClusterState(204): Hosts are {2998b885bdef=0} racks are {/default-rack=0} 2024-11-13T07:16:37,943 DEBUG [2998b885bdef:44607 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T07:16:37,943 DEBUG [2998b885bdef:44607 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T07:16:37,944 DEBUG [2998b885bdef:44607 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T07:16:37,944 DEBUG [2998b885bdef:44607 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T07:16:37,944 INFO [2998b885bdef:44607 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T07:16:37,944 INFO [2998b885bdef:44607 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T07:16:37,944 DEBUG [2998b885bdef:44607 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T07:16:37,944 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=51 updating hbase:meta row=56b34ea82217cf90ce43d525dd7e1a9d, regionState=OPENING, regionLocation=2998b885bdef,36543,1731482143970 2024-11-13T07:16:37,945 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=1f49ca9e4320a113a985ca82c73f5c2a, regionState=OPENING, regionLocation=2998b885bdef,36543,1731482143970 2024-11-13T07:16:37,946 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=49 updating hbase:meta row=ff8b864839731879b51af65fe2a46ac9, regionState=OPENING, regionLocation=2998b885bdef,41365,1731482143808 2024-11-13T07:16:37,961 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=46, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=56b34ea82217cf90ce43d525dd7e1a9d, ASSIGN because future has completed 2024-11-13T07:16:37,961 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56b34ea82217cf90ce43d525dd7e1a9d, server=2998b885bdef,36543,1731482143970}] 2024-11-13T07:16:37,969 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=46, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=1f49ca9e4320a113a985ca82c73f5c2a, ASSIGN because future has completed 2024-11-13T07:16:37,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=50, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1f49ca9e4320a113a985ca82c73f5c2a, server=2998b885bdef,36543,1731482143970}] 2024-11-13T07:16:37,973 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=46, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=SPTestTable, region=ff8b864839731879b51af65fe2a46ac9, ASSIGN because future has completed 2024-11-13T07:16:37,975 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=49, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff8b864839731879b51af65fe2a46ac9, server=2998b885bdef,41365,1731482143808}] 2024-11-13T07:16:43,484 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T07:16:45,132 INFO [regionserver/2998b885bdef:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: ExecutorStatusChore was stopped 2024-11-13T07:16:47,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotVerifyCallable, pid=45 2024-11-13T07:16:47,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotVerifyCallable, pid=40 2024-11-13T07:16:47,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotVerifyCallable, pid=36 2024-11-13T07:16:47,730 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d. 2024-11-13T07:16:47,730 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 56b34ea82217cf90ce43d525dd7e1a9d, NAME => 'SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d.', STARTKEY => 'b333332f', ENDKEY => 'ccccccc8'} 2024-11-13T07:16:47,730 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table SPTestTable 56b34ea82217cf90ce43d525dd7e1a9d 2024-11-13T07:16:47,730 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:16:47,730 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 56b34ea82217cf90ce43d525dd7e1a9d 2024-11-13T07:16:47,730 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 56b34ea82217cf90ce43d525dd7e1a9d 2024-11-13T07:16:47,734 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_VERIFY_SNAPSHOT, pid=40}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=40 2024-11-13T07:16:47,734 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 {event_type=RS_VERIFY_SNAPSHOT, pid=36}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=36 2024-11-13T07:16:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] master.HMaster(4169): Remote procedure done, pid=40 2024-11-13T07:16:47,739 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_VERIFY_SNAPSHOT, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-13T07:16:47,744 INFO [StoreOpener-56b34ea82217cf90ce43d525dd7e1a9d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region 56b34ea82217cf90ce43d525dd7e1a9d 2024-11-13T07:16:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] master.HMaster(4169): Remote procedure done, pid=36 2024-11-13T07:16:47,749 INFO [StoreOpener-56b34ea82217cf90ce43d525dd7e1a9d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56b34ea82217cf90ce43d525dd7e1a9d columnFamilyName cf 2024-11-13T07:16:47,749 DEBUG [StoreOpener-56b34ea82217cf90ce43d525dd7e1a9d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:16:47,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41365 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotVerifyCallable, pid=39 2024-11-13T07:16:47,752 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1473): pid=40, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,36543,1731482143970 Release verify snapshot worker=2998b885bdef,36543,1731482143970 2024-11-13T07:16:47,752 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9. 2024-11-13T07:16:47,753 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => ff8b864839731879b51af65fe2a46ac9, NAME => 'SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9.', STARTKEY => '4ccccccb', ENDKEY => '66666664'} 2024-11-13T07:16:47,753 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table SPTestTable ff8b864839731879b51af65fe2a46ac9 2024-11-13T07:16:47,753 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:16:47,753 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for ff8b864839731879b51af65fe2a46ac9 2024-11-13T07:16:47,753 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for ff8b864839731879b51af65fe2a46ac9 2024-11-13T07:16:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44607 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-13T07:16:47,756 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 {event_type=RS_VERIFY_SNAPSHOT, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-13T07:16:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44607 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-13T07:16:47,759 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=25, state=SUCCESS, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,36543,1731482143970 in 20.1070 sec 2024-11-13T07:16:47,759 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1473): pid=36, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,36543,1731482143970 Release verify snapshot worker=2998b885bdef,36543,1731482143970 2024-11-13T07:16:47,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=25, state=SUCCESS, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,36543,1731482143970 in 20.1140 sec 2024-11-13T07:16:47,772 INFO [StoreOpener-ff8b864839731879b51af65fe2a46ac9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region ff8b864839731879b51af65fe2a46ac9 2024-11-13T07:16:47,774 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1473): pid=45, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,36543,1731482143970 Release verify snapshot worker=2998b885bdef,36543,1731482143970 2024-11-13T07:16:47,774 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1473): pid=39, ppid=25, state=RUNNABLE, hasLock=true; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,41365,1731482143808 Release verify snapshot worker=2998b885bdef,41365,1731482143808 2024-11-13T07:16:47,780 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=25, state=SUCCESS, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,36543,1731482143970 in 20.1290 sec 2024-11-13T07:16:47,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=25 2024-11-13T07:16:47,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=25, state=SUCCESS, hasLock=false; SnapshotVerifyProcedure, snapshot=SnapshotProcedureTest, targetServer=2998b885bdef,41365,1731482143808 in 20.1290 sec 2024-11-13T07:16:47,783 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=25, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=25, snapshot={ ss=SnapshotProcedureTest table=SPTestTable type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2024-11-13T07:16:47,786 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(414): Sentinel is done, just moving the snapshot from hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/.hbase-snapshot/.tmp/SnapshotProcedureTest to hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/.hbase-snapshot/SnapshotProcedureTest 2024-11-13T07:16:47,786 INFO [StoreOpener-ff8b864839731879b51af65fe2a46ac9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff8b864839731879b51af65fe2a46ac9 columnFamilyName cf 2024-11-13T07:16:47,786 DEBUG [StoreOpener-ff8b864839731879b51af65fe2a46ac9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:16:47,787 INFO [StoreOpener-ff8b864839731879b51af65fe2a46ac9-1 {}] regionserver.HStore(327): Store=ff8b864839731879b51af65fe2a46ac9/cf, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:16:47,787 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for ff8b864839731879b51af65fe2a46ac9 2024-11-13T07:16:47,789 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=25, state=RUNNABLE:SNAPSHOT_POST_OPERATION, hasLock=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=25, snapshot={ ss=SnapshotProcedureTest table=SPTestTable type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2024-11-13T07:16:47,789 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1451): unregister snapshot={ ss=SnapshotProcedureTest table=SPTestTable type=FLUSH ttl=0 }, snapshot procedure id = 25 2024-11-13T07:16:47,790 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/ff8b864839731879b51af65fe2a46ac9 2024-11-13T07:16:47,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=25, snapshot={ ss=SnapshotProcedureTest table=SPTestTable type=FLUSH ttl=0 } in 30.9190 sec 2024-11-13T07:16:47,794 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/ff8b864839731879b51af65fe2a46ac9 2024-11-13T07:16:47,794 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for ff8b864839731879b51af65fe2a46ac9 2024-11-13T07:16:47,794 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for ff8b864839731879b51af65fe2a46ac9 2024-11-13T07:16:47,799 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for ff8b864839731879b51af65fe2a46ac9 2024-11-13T07:16:47,803 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened ff8b864839731879b51af65fe2a46ac9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69571039, jitterRate=0.03668926656246185}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:16:47,803 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ff8b864839731879b51af65fe2a46ac9 2024-11-13T07:16:47,803 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for ff8b864839731879b51af65fe2a46ac9: Running coprocessor pre-open hook at 1731482207753Writing region info on filesystem at 1731482207753Initializing all the Stores at 1731482207764 (+11 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482207764Cleaning up temporary data from old regions at 1731482207794 (+30 ms)Running coprocessor post-open hooks at 1731482207803 (+9 ms)Region opened successfully at 1731482207803 2024-11-13T07:16:47,805 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9., pid=54, masterSystemTime=1731482207742 2024-11-13T07:16:47,808 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9. 2024-11-13T07:16:47,808 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9. 2024-11-13T07:16:47,809 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=49 updating hbase:meta row=ff8b864839731879b51af65fe2a46ac9, regionState=OPEN, openSeqNum=2, regionLocation=2998b885bdef,41365,1731482143808 2024-11-13T07:16:47,810 DEBUG [StoreOpener-56b34ea82217cf90ce43d525dd7e1a9d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/56b34ea82217cf90ce43d525dd7e1a9d/cf/a5ea41c972c64a38a4d9030c966bf2bf 2024-11-13T07:16:47,811 INFO [StoreOpener-56b34ea82217cf90ce43d525dd7e1a9d-1 {}] regionserver.HStore(327): Store=56b34ea82217cf90ce43d525dd7e1a9d/cf, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:16:47,811 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 56b34ea82217cf90ce43d525dd7e1a9d 2024-11-13T07:16:47,812 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/56b34ea82217cf90ce43d525dd7e1a9d 2024-11-13T07:16:47,816 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/56b34ea82217cf90ce43d525dd7e1a9d 2024-11-13T07:16:47,817 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=49, state=RUNNABLE, hasLock=false; OpenRegionProcedure ff8b864839731879b51af65fe2a46ac9, server=2998b885bdef,41365,1731482143808 because future has completed 2024-11-13T07:16:47,817 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 56b34ea82217cf90ce43d525dd7e1a9d 2024-11-13T07:16:47,817 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 56b34ea82217cf90ce43d525dd7e1a9d 2024-11-13T07:16:47,821 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 56b34ea82217cf90ce43d525dd7e1a9d 2024-11-13T07:16:47,825 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=49 2024-11-13T07:16:47,825 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=49, state=SUCCESS, hasLock=false; OpenRegionProcedure ff8b864839731879b51af65fe2a46ac9, server=2998b885bdef,41365,1731482143808 in 9.8440 sec 2024-11-13T07:16:47,829 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=46, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=ff8b864839731879b51af65fe2a46ac9, ASSIGN in 10.0360 sec 2024-11-13T07:16:47,842 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/56b34ea82217cf90ce43d525dd7e1a9d/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-13T07:16:47,844 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 56b34ea82217cf90ce43d525dd7e1a9d; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60182669, jitterRate=-0.10320834815502167}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:16:47,844 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 56b34ea82217cf90ce43d525dd7e1a9d 2024-11-13T07:16:47,845 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 56b34ea82217cf90ce43d525dd7e1a9d: Running coprocessor pre-open hook at 1731482207731Writing region info on filesystem at 1731482207731Initializing all the Stores at 1731482207738 (+7 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482207738Cleaning up temporary data from old regions at 1731482207817 (+79 ms)Running coprocessor post-open hooks at 1731482207844 (+27 ms)Region opened successfully at 1731482207844 2024-11-13T07:16:47,848 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d., pid=52, masterSystemTime=1731482207715 2024-11-13T07:16:47,851 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d. 2024-11-13T07:16:47,852 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d. 2024-11-13T07:16:47,857 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a. 2024-11-13T07:16:47,857 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 1f49ca9e4320a113a985ca82c73f5c2a, NAME => 'SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a.', STARTKEY => '19999999', ENDKEY => '33333332'} 2024-11-13T07:16:47,857 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table SPTestTable 1f49ca9e4320a113a985ca82c73f5c2a 2024-11-13T07:16:47,858 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T07:16:47,858 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 1f49ca9e4320a113a985ca82c73f5c2a 2024-11-13T07:16:47,858 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 1f49ca9e4320a113a985ca82c73f5c2a 2024-11-13T07:16:47,860 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=51 updating hbase:meta row=56b34ea82217cf90ce43d525dd7e1a9d, regionState=OPEN, openSeqNum=6, regionLocation=2998b885bdef,36543,1731482143970 2024-11-13T07:16:47,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=51, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56b34ea82217cf90ce43d525dd7e1a9d, server=2998b885bdef,36543,1731482143970 because future has completed 2024-11-13T07:16:47,878 INFO [StoreOpener-1f49ca9e4320a113a985ca82c73f5c2a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf of region 1f49ca9e4320a113a985ca82c73f5c2a 2024-11-13T07:16:47,880 INFO [StoreOpener-1f49ca9e4320a113a985ca82c73f5c2a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f49ca9e4320a113a985ca82c73f5c2a columnFamilyName cf 2024-11-13T07:16:47,880 DEBUG [StoreOpener-1f49ca9e4320a113a985ca82c73f5c2a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T07:16:47,885 INFO [StoreOpener-1f49ca9e4320a113a985ca82c73f5c2a-1 {}] regionserver.HStore(327): Store=1f49ca9e4320a113a985ca82c73f5c2a/cf, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T07:16:47,885 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 1f49ca9e4320a113a985ca82c73f5c2a 2024-11-13T07:16:47,886 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/1f49ca9e4320a113a985ca82c73f5c2a 2024-11-13T07:16:47,889 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/1f49ca9e4320a113a985ca82c73f5c2a 2024-11-13T07:16:47,890 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 1f49ca9e4320a113a985ca82c73f5c2a 2024-11-13T07:16:47,890 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 1f49ca9e4320a113a985ca82c73f5c2a 2024-11-13T07:16:47,895 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 1f49ca9e4320a113a985ca82c73f5c2a 2024-11-13T07:16:47,896 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 1f49ca9e4320a113a985ca82c73f5c2a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67906318, jitterRate=0.011882990598678589}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T07:16:47,896 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1f49ca9e4320a113a985ca82c73f5c2a 2024-11-13T07:16:47,896 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 1f49ca9e4320a113a985ca82c73f5c2a: Running coprocessor pre-open hook at 1731482207858Writing region info on filesystem at 1731482207858Initializing all the Stores at 1731482207866 (+8 ms)Instantiating store for column family {NAME => 'cf', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731482207866Cleaning up temporary data from old regions at 1731482207890 (+24 ms)Running coprocessor post-open hooks at 1731482207896 (+6 ms)Region opened successfully at 1731482207896 2024-11-13T07:16:47,898 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a., pid=53, masterSystemTime=1731482207715 2024-11-13T07:16:47,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=51 2024-11-13T07:16:47,904 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=51, state=SUCCESS, hasLock=false; OpenRegionProcedure 56b34ea82217cf90ce43d525dd7e1a9d, server=2998b885bdef,36543,1731482143970 in 9.9160 sec 2024-11-13T07:16:47,906 DEBUG [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a. 2024-11-13T07:16:47,906 INFO [RS_OPEN_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a. 2024-11-13T07:16:47,920 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=1f49ca9e4320a113a985ca82c73f5c2a, regionState=OPEN, openSeqNum=2, regionLocation=2998b885bdef,36543,1731482143970 2024-11-13T07:16:47,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=46, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=56b34ea82217cf90ce43d525dd7e1a9d, ASSIGN in 10.1150 sec 2024-11-13T07:16:47,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-10-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=50, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1f49ca9e4320a113a985ca82c73f5c2a, server=2998b885bdef,36543,1731482143970 because future has completed 2024-11-13T07:16:47,940 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44607 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=2998b885bdef,36543,1731482143970, table=SPTestTable, region=1f49ca9e4320a113a985ca82c73f5c2a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T07:16:47,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=50 2024-11-13T07:16:47,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=50, state=SUCCESS, hasLock=false; OpenRegionProcedure 1f49ca9e4320a113a985ca82c73f5c2a, server=2998b885bdef,36543,1731482143970 in 9.9700 sec 2024-11-13T07:16:47,966 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 2998b885bdef,39195,1731482143882 after splitting done 2024-11-13T07:16:47,967 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=46 2024-11-13T07:16:47,968 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=46, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=SPTestTable, region=1f49ca9e4320a113a985ca82c73f5c2a, ASSIGN in 10.1640 sec 2024-11-13T07:16:47,973 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; ServerCrashProcedure 2998b885bdef,39195,1731482143882, splitWal=true, meta=false in 20.1880 sec 2024-11-13T07:16:48,071 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=SPTestTable,, stopping at row=SPTestTable ,, for max=2147483647 with caching=100 2024-11-13T07:16:48,078 WARN [Time-limited test {}] procedure2.ProcedureTestingUtility(193): Set Kill before store update to: false 2024-11-13T07:16:48,078 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T07:16:48,078 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T07:16:48,078 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.procedure.TestSnapshotProcedure.teardown(TestSnapshotProcedure.java:151) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T07:16:48,078 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:16:48,078 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:16:48,078 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T07:16:48,078 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1191204059, stopped=false 2024-11-13T07:16:48,078 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=2998b885bdef,44607,1731482143517 2024-11-13T07:16:48,079 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T07:16:48,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T07:16:48,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:16:48,129 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T07:16:48,129 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T07:16:48,129 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.procedure.TestSnapshotProcedure.teardown(TestSnapshotProcedure.java:151) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T07:16:48,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:16:48,129 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2998b885bdef,41365,1731482143808' ***** 2024-11-13T07:16:48,130 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T07:16:48,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36543-0x101330cc6ab0003, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T07:16:48,130 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2998b885bdef,36543,1731482143970' ***** 2024-11-13T07:16:48,130 DEBUG [pool-309-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41365-0x101330cc6ab0001, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T07:16:48,130 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T07:16:48,130 DEBUG [pool-309-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41365-0x101330cc6ab0001, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:16:48,130 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T07:16:48,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36543-0x101330cc6ab0003, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:16:48,130 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41365-0x101330cc6ab0001, quorum=127.0.0.1:63253, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T07:16:48,131 INFO [RS:0;2998b885bdef:41365 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T07:16:48,131 INFO [RS:0;2998b885bdef:41365 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T07:16:48,131 INFO [RS:0;2998b885bdef:41365 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T07:16:48,131 INFO [RS:0;2998b885bdef:41365 {}] regionserver.HRegionServer(3091): Received CLOSE for 2e4b72fe1d04f499cf9c51ce52c79100 2024-11-13T07:16:48,132 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T07:16:48,132 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36543-0x101330cc6ab0003, quorum=127.0.0.1:63253, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T07:16:48,135 INFO [RS:2;2998b885bdef:36543 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T07:16:48,135 INFO [RS:2;2998b885bdef:36543 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T07:16:48,135 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T07:16:48,135 INFO [RS:2;2998b885bdef:36543 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T07:16:48,135 INFO [RS:2;2998b885bdef:36543 {}] regionserver.HRegionServer(3091): Received CLOSE for dcdab597c439e333702f5a0384e4db9a 2024-11-13T07:16:48,140 INFO [RS:0;2998b885bdef:41365 {}] regionserver.HRegionServer(3091): Received CLOSE for ff8b864839731879b51af65fe2a46ac9 2024-11-13T07:16:48,140 INFO [RS:0;2998b885bdef:41365 {}] regionserver.HRegionServer(3091): Received CLOSE for 1357138ab69f6578d72ce8d295f98b00 2024-11-13T07:16:48,141 INFO [RS:0;2998b885bdef:41365 {}] regionserver.HRegionServer(3091): Received CLOSE for d029c66cacb99cb6bcc28a1f47b21331 2024-11-13T07:16:48,141 INFO [RS:0;2998b885bdef:41365 {}] regionserver.HRegionServer(3091): Received CLOSE for dc0d3e1b8b1a5a44332ecc1f0118002b 2024-11-13T07:16:48,141 INFO [RS:0;2998b885bdef:41365 {}] regionserver.HRegionServer(959): stopping server 2998b885bdef,41365,1731482143808 2024-11-13T07:16:48,141 INFO [RS:0;2998b885bdef:41365 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T07:16:48,141 INFO [RS:0;2998b885bdef:41365 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;2998b885bdef:41365. 2024-11-13T07:16:48,141 DEBUG [RS:0;2998b885bdef:41365 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T07:16:48,141 DEBUG [RS:0;2998b885bdef:41365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:16:48,141 INFO [RS:0;2998b885bdef:41365 {}] regionserver.HRegionServer(1321): Waiting on 5 regions to close 2024-11-13T07:16:48,141 DEBUG [RS:0;2998b885bdef:41365 {}] regionserver.HRegionServer(1325): Online Regions={2e4b72fe1d04f499cf9c51ce52c79100=SPTestTable,,1731482155666.2e4b72fe1d04f499cf9c51ce52c79100., ff8b864839731879b51af65fe2a46ac9=SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9., 1357138ab69f6578d72ce8d295f98b00=SPTestTable,66666664,1731482155666.1357138ab69f6578d72ce8d295f98b00., d029c66cacb99cb6bcc28a1f47b21331=SPTestTable,ccccccc8,1731482155666.d029c66cacb99cb6bcc28a1f47b21331., dc0d3e1b8b1a5a44332ecc1f0118002b=SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b.} 2024-11-13T07:16:48,141 DEBUG [RS:0;2998b885bdef:41365 {}] regionserver.HRegionServer(1351): Waiting on 1357138ab69f6578d72ce8d295f98b00, 2e4b72fe1d04f499cf9c51ce52c79100, d029c66cacb99cb6bcc28a1f47b21331, dc0d3e1b8b1a5a44332ecc1f0118002b, ff8b864839731879b51af65fe2a46ac9 2024-11-13T07:16:48,142 INFO [RS:2;2998b885bdef:36543 {}] regionserver.HRegionServer(3091): Received CLOSE for 9e8af980aa626886ac3cb035f55a2e4c 2024-11-13T07:16:48,142 INFO [RS:2;2998b885bdef:36543 {}] regionserver.HRegionServer(3091): Received CLOSE for 045378476123254a5c2616896d4edfcf 2024-11-13T07:16:48,142 INFO [RS:2;2998b885bdef:36543 {}] regionserver.HRegionServer(3091): Received CLOSE for 56b34ea82217cf90ce43d525dd7e1a9d 2024-11-13T07:16:48,142 INFO [RS:2;2998b885bdef:36543 {}] regionserver.HRegionServer(3091): Received CLOSE for 1f49ca9e4320a113a985ca82c73f5c2a 2024-11-13T07:16:48,142 INFO [RS:2;2998b885bdef:36543 {}] regionserver.HRegionServer(959): stopping server 2998b885bdef,36543,1731482143970 2024-11-13T07:16:48,142 INFO [RS:2;2998b885bdef:36543 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T07:16:48,142 INFO [RS:2;2998b885bdef:36543 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;2998b885bdef:36543. 2024-11-13T07:16:48,142 DEBUG [RS:2;2998b885bdef:36543 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T07:16:48,142 DEBUG [RS:2;2998b885bdef:36543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:16:48,143 INFO [RS:2;2998b885bdef:36543 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T07:16:48,143 INFO [RS:2;2998b885bdef:36543 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T07:16:48,143 INFO [RS:2;2998b885bdef:36543 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T07:16:48,143 INFO [RS:2;2998b885bdef:36543 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T07:16:48,144 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2e4b72fe1d04f499cf9c51ce52c79100, disabling compactions & flushes 2024-11-13T07:16:48,144 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region SPTestTable,,1731482155666.2e4b72fe1d04f499cf9c51ce52c79100. 2024-11-13T07:16:48,144 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,,1731482155666.2e4b72fe1d04f499cf9c51ce52c79100. 2024-11-13T07:16:48,144 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,,1731482155666.2e4b72fe1d04f499cf9c51ce52c79100. after waiting 0 ms 2024-11-13T07:16:48,144 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,,1731482155666.2e4b72fe1d04f499cf9c51ce52c79100. 2024-11-13T07:16:48,147 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dcdab597c439e333702f5a0384e4db9a, disabling compactions & flushes 2024-11-13T07:16:48,147 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region SPTestTable,e6666661,1731482155666.dcdab597c439e333702f5a0384e4db9a. 2024-11-13T07:16:48,147 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,e6666661,1731482155666.dcdab597c439e333702f5a0384e4db9a. 2024-11-13T07:16:48,147 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,e6666661,1731482155666.dcdab597c439e333702f5a0384e4db9a. after waiting 0 ms 2024-11-13T07:16:48,147 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,e6666661,1731482155666.dcdab597c439e333702f5a0384e4db9a. 2024-11-13T07:16:48,147 INFO [RS:2;2998b885bdef:36543 {}] regionserver.HRegionServer(1321): Waiting on 6 regions to close 2024-11-13T07:16:48,147 DEBUG [RS:2;2998b885bdef:36543 {}] regionserver.HRegionServer(1325): Online Regions={dcdab597c439e333702f5a0384e4db9a=SPTestTable,e6666661,1731482155666.dcdab597c439e333702f5a0384e4db9a., 9e8af980aa626886ac3cb035f55a2e4c=SPTestTable,7ffffffd,1731482155666.9e8af980aa626886ac3cb035f55a2e4c., 045378476123254a5c2616896d4edfcf=SPTestTable,33333332,1731482155666.045378476123254a5c2616896d4edfcf., 56b34ea82217cf90ce43d525dd7e1a9d=SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d., 1588230740=hbase:meta,,1.1588230740, 1f49ca9e4320a113a985ca82c73f5c2a=SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a.} 2024-11-13T07:16:48,147 DEBUG [RS:2;2998b885bdef:36543 {}] regionserver.HRegionServer(1351): Waiting on 045378476123254a5c2616896d4edfcf, 1588230740, 1f49ca9e4320a113a985ca82c73f5c2a, 56b34ea82217cf90ce43d525dd7e1a9d, 9e8af980aa626886ac3cb035f55a2e4c, dcdab597c439e333702f5a0384e4db9a 2024-11-13T07:16:48,148 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T07:16:48,148 INFO [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T07:16:48,148 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T07:16:48,148 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T07:16:48,148 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T07:16:48,149 INFO [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.38 KB heapSize=25.80 KB 2024-11-13T07:16:48,177 INFO [regionserver/2998b885bdef:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T07:16:48,177 INFO [regionserver/2998b885bdef:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T07:16:48,187 INFO [regionserver/2998b885bdef:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T07:16:48,187 INFO [regionserver/2998b885bdef:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T07:16:48,198 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/dcdab597c439e333702f5a0384e4db9a/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T07:16:48,199 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed SPTestTable,e6666661,1731482155666.dcdab597c439e333702f5a0384e4db9a. 2024-11-13T07:16:48,199 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dcdab597c439e333702f5a0384e4db9a: Waiting for close lock at 1731482208147Running coprocessor pre-close hooks at 1731482208147Disabling compacts and flushes for region at 1731482208147Disabling writes for close at 1731482208147Writing region close event to WAL at 1731482208176 (+29 ms)Running coprocessor post-close hooks at 1731482208199 (+23 ms)Closed at 1731482208199 2024-11-13T07:16:48,199 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed SPTestTable,e6666661,1731482155666.dcdab597c439e333702f5a0384e4db9a. 2024-11-13T07:16:48,199 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9e8af980aa626886ac3cb035f55a2e4c, disabling compactions & flushes 2024-11-13T07:16:48,199 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region SPTestTable,7ffffffd,1731482155666.9e8af980aa626886ac3cb035f55a2e4c. 2024-11-13T07:16:48,199 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,7ffffffd,1731482155666.9e8af980aa626886ac3cb035f55a2e4c. 2024-11-13T07:16:48,200 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,7ffffffd,1731482155666.9e8af980aa626886ac3cb035f55a2e4c. after waiting 0 ms 2024-11-13T07:16:48,200 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,7ffffffd,1731482155666.9e8af980aa626886ac3cb035f55a2e4c. 2024-11-13T07:16:48,207 INFO [regionserver/2998b885bdef:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T07:16:48,213 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/hbase/meta/1588230740/.tmp/info/6e2b203e717c4150abce7792f0de1fc4 is 167, key is SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a./info:regioninfo/1731482207920/Put/seqid=0 2024-11-13T07:16:48,217 INFO [regionserver/2998b885bdef:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T07:16:48,226 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/2e4b72fe1d04f499cf9c51ce52c79100/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-13T07:16:48,227 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed SPTestTable,,1731482155666.2e4b72fe1d04f499cf9c51ce52c79100. 2024-11-13T07:16:48,227 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2e4b72fe1d04f499cf9c51ce52c79100: Waiting for close lock at 1731482208144Running coprocessor pre-close hooks at 1731482208144Disabling compacts and flushes for region at 1731482208144Disabling writes for close at 1731482208144Writing region close event to WAL at 1731482208192 (+48 ms)Running coprocessor post-close hooks at 1731482208227 (+35 ms)Closed at 1731482208227 2024-11-13T07:16:48,228 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed SPTestTable,,1731482155666.2e4b72fe1d04f499cf9c51ce52c79100. 2024-11-13T07:16:48,228 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ff8b864839731879b51af65fe2a46ac9, disabling compactions & flushes 2024-11-13T07:16:48,228 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9. 2024-11-13T07:16:48,228 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9. 2024-11-13T07:16:48,228 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9. after waiting 0 ms 2024-11-13T07:16:48,228 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9. 2024-11-13T07:16:48,256 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/9e8af980aa626886ac3cb035f55a2e4c/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-13T07:16:48,257 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed SPTestTable,7ffffffd,1731482155666.9e8af980aa626886ac3cb035f55a2e4c. 2024-11-13T07:16:48,258 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9e8af980aa626886ac3cb035f55a2e4c: Waiting for close lock at 1731482208199Running coprocessor pre-close hooks at 1731482208199Disabling compacts and flushes for region at 1731482208199Disabling writes for close at 1731482208200 (+1 ms)Writing region close event to WAL at 1731482208208 (+8 ms)Running coprocessor post-close hooks at 1731482208257 (+49 ms)Closed at 1731482208257 2024-11-13T07:16:48,258 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed SPTestTable,7ffffffd,1731482155666.9e8af980aa626886ac3cb035f55a2e4c. 2024-11-13T07:16:48,258 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 045378476123254a5c2616896d4edfcf, disabling compactions & flushes 2024-11-13T07:16:48,258 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region SPTestTable,33333332,1731482155666.045378476123254a5c2616896d4edfcf. 2024-11-13T07:16:48,258 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,33333332,1731482155666.045378476123254a5c2616896d4edfcf. 2024-11-13T07:16:48,258 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,33333332,1731482155666.045378476123254a5c2616896d4edfcf. after waiting 0 ms 2024-11-13T07:16:48,258 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,33333332,1731482155666.045378476123254a5c2616896d4edfcf. 2024-11-13T07:16:48,289 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/ff8b864839731879b51af65fe2a46ac9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T07:16:48,290 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9. 2024-11-13T07:16:48,290 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ff8b864839731879b51af65fe2a46ac9: Waiting for close lock at 1731482208228Running coprocessor pre-close hooks at 1731482208228Disabling compacts and flushes for region at 1731482208228Disabling writes for close at 1731482208228Writing region close event to WAL at 1731482208268 (+40 ms)Running coprocessor post-close hooks at 1731482208290 (+22 ms)Closed at 1731482208290 2024-11-13T07:16:48,290 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed SPTestTable,4ccccccb,1731482155666.ff8b864839731879b51af65fe2a46ac9. 2024-11-13T07:16:48,290 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1357138ab69f6578d72ce8d295f98b00, disabling compactions & flushes 2024-11-13T07:16:48,290 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region SPTestTable,66666664,1731482155666.1357138ab69f6578d72ce8d295f98b00. 2024-11-13T07:16:48,290 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,66666664,1731482155666.1357138ab69f6578d72ce8d295f98b00. 2024-11-13T07:16:48,290 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,66666664,1731482155666.1357138ab69f6578d72ce8d295f98b00. after waiting 0 ms 2024-11-13T07:16:48,290 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,66666664,1731482155666.1357138ab69f6578d72ce8d295f98b00. 2024-11-13T07:16:48,301 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/045378476123254a5c2616896d4edfcf/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-13T07:16:48,305 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed SPTestTable,33333332,1731482155666.045378476123254a5c2616896d4edfcf. 2024-11-13T07:16:48,305 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 045378476123254a5c2616896d4edfcf: Waiting for close lock at 1731482208258Running coprocessor pre-close hooks at 1731482208258Disabling compacts and flushes for region at 1731482208258Disabling writes for close at 1731482208258Writing region close event to WAL at 1731482208281 (+23 ms)Running coprocessor post-close hooks at 1731482208305 (+24 ms)Closed at 1731482208305 2024-11-13T07:16:48,305 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed SPTestTable,33333332,1731482155666.045378476123254a5c2616896d4edfcf. 2024-11-13T07:16:48,305 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 56b34ea82217cf90ce43d525dd7e1a9d, disabling compactions & flushes 2024-11-13T07:16:48,305 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d. 2024-11-13T07:16:48,305 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d. 2024-11-13T07:16:48,305 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d. after waiting 0 ms 2024-11-13T07:16:48,305 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d. 2024-11-13T07:16:48,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741864_1040 (size=19817) 2024-11-13T07:16:48,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741864_1040 (size=19817) 2024-11-13T07:16:48,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741864_1040 (size=19817) 2024-11-13T07:16:48,332 INFO [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.22 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/hbase/meta/1588230740/.tmp/info/6e2b203e717c4150abce7792f0de1fc4 2024-11-13T07:16:48,348 DEBUG [RS:2;2998b885bdef:36543 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1f49ca9e4320a113a985ca82c73f5c2a, 56b34ea82217cf90ce43d525dd7e1a9d 2024-11-13T07:16:48,348 DEBUG [RS:0;2998b885bdef:41365 {}] regionserver.HRegionServer(1351): Waiting on 1357138ab69f6578d72ce8d295f98b00, d029c66cacb99cb6bcc28a1f47b21331, dc0d3e1b8b1a5a44332ecc1f0118002b 2024-11-13T07:16:48,349 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/1357138ab69f6578d72ce8d295f98b00/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-13T07:16:48,351 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed SPTestTable,66666664,1731482155666.1357138ab69f6578d72ce8d295f98b00. 2024-11-13T07:16:48,352 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1357138ab69f6578d72ce8d295f98b00: Waiting for close lock at 1731482208290Running coprocessor pre-close hooks at 1731482208290Disabling compacts and flushes for region at 1731482208290Disabling writes for close at 1731482208290Writing region close event to WAL at 1731482208299 (+9 ms)Running coprocessor post-close hooks at 1731482208351 (+52 ms)Closed at 1731482208351 2024-11-13T07:16:48,352 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed SPTestTable,66666664,1731482155666.1357138ab69f6578d72ce8d295f98b00. 2024-11-13T07:16:48,352 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d029c66cacb99cb6bcc28a1f47b21331, disabling compactions & flushes 2024-11-13T07:16:48,352 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region SPTestTable,ccccccc8,1731482155666.d029c66cacb99cb6bcc28a1f47b21331. 2024-11-13T07:16:48,352 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,ccccccc8,1731482155666.d029c66cacb99cb6bcc28a1f47b21331. 2024-11-13T07:16:48,352 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,ccccccc8,1731482155666.d029c66cacb99cb6bcc28a1f47b21331. after waiting 0 ms 2024-11-13T07:16:48,352 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,ccccccc8,1731482155666.d029c66cacb99cb6bcc28a1f47b21331. 2024-11-13T07:16:48,364 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/56b34ea82217cf90ce43d525dd7e1a9d/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=5 2024-11-13T07:16:48,365 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d. 2024-11-13T07:16:48,365 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 56b34ea82217cf90ce43d525dd7e1a9d: Waiting for close lock at 1731482208305Running coprocessor pre-close hooks at 1731482208305Disabling compacts and flushes for region at 1731482208305Disabling writes for close at 1731482208305Writing region close event to WAL at 1731482208340 (+35 ms)Running coprocessor post-close hooks at 1731482208365 (+25 ms)Closed at 1731482208365 2024-11-13T07:16:48,366 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed SPTestTable,b333332f,1731482155666.56b34ea82217cf90ce43d525dd7e1a9d. 2024-11-13T07:16:48,366 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1f49ca9e4320a113a985ca82c73f5c2a, disabling compactions & flushes 2024-11-13T07:16:48,366 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a. 2024-11-13T07:16:48,366 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a. 2024-11-13T07:16:48,366 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a. after waiting 0 ms 2024-11-13T07:16:48,366 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a. 2024-11-13T07:16:48,397 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/hbase/meta/1588230740/.tmp/ns/b2955d14b8ee482bb06daa09100eff24 is 43, key is default/ns:d/1731482155593/Put/seqid=0 2024-11-13T07:16:48,417 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/1f49ca9e4320a113a985ca82c73f5c2a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T07:16:48,421 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a. 2024-11-13T07:16:48,421 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1f49ca9e4320a113a985ca82c73f5c2a: Waiting for close lock at 1731482208366Running coprocessor pre-close hooks at 1731482208366Disabling compacts and flushes for region at 1731482208366Disabling writes for close at 1731482208366Writing region close event to WAL at 1731482208376 (+10 ms)Running coprocessor post-close hooks at 1731482208421 (+45 ms)Closed at 1731482208421 2024-11-13T07:16:48,421 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed SPTestTable,19999999,1731482155666.1f49ca9e4320a113a985ca82c73f5c2a. 2024-11-13T07:16:48,460 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/d029c66cacb99cb6bcc28a1f47b21331/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T07:16:48,463 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed SPTestTable,ccccccc8,1731482155666.d029c66cacb99cb6bcc28a1f47b21331. 2024-11-13T07:16:48,463 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d029c66cacb99cb6bcc28a1f47b21331: Waiting for close lock at 1731482208352Running coprocessor pre-close hooks at 1731482208352Disabling compacts and flushes for region at 1731482208352Disabling writes for close at 1731482208352Writing region close event to WAL at 1731482208421 (+69 ms)Running coprocessor post-close hooks at 1731482208463 (+42 ms)Closed at 1731482208463 2024-11-13T07:16:48,463 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed SPTestTable,ccccccc8,1731482155666.d029c66cacb99cb6bcc28a1f47b21331. 2024-11-13T07:16:48,463 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dc0d3e1b8b1a5a44332ecc1f0118002b, disabling compactions & flushes 2024-11-13T07:16:48,464 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b. 2024-11-13T07:16:48,464 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b. 2024-11-13T07:16:48,464 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b. after waiting 0 ms 2024-11-13T07:16:48,464 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b. 2024-11-13T07:16:48,479 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/default/SPTestTable/dc0d3e1b8b1a5a44332ecc1f0118002b/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T07:16:48,480 INFO [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b. 2024-11-13T07:16:48,481 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dc0d3e1b8b1a5a44332ecc1f0118002b: Waiting for close lock at 1731482208463Running coprocessor pre-close hooks at 1731482208463Disabling compacts and flushes for region at 1731482208463Disabling writes for close at 1731482208464 (+1 ms)Writing region close event to WAL at 1731482208467 (+3 ms)Running coprocessor post-close hooks at 1731482208480 (+13 ms)Closed at 1731482208480 2024-11-13T07:16:48,481 DEBUG [RS_CLOSE_REGION-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed SPTestTable,99999996,1731482155666.dc0d3e1b8b1a5a44332ecc1f0118002b. 2024-11-13T07:16:48,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741865_1041 (size=5153) 2024-11-13T07:16:48,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741865_1041 (size=5153) 2024-11-13T07:16:48,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741865_1041 (size=5153) 2024-11-13T07:16:48,551 DEBUG [RS:2;2998b885bdef:36543 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T07:16:48,551 INFO [RS:0;2998b885bdef:41365 {}] regionserver.HRegionServer(976): stopping server 2998b885bdef,41365,1731482143808; all regions closed. 2024-11-13T07:16:48,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741834_1010 (size=4022) 2024-11-13T07:16:48,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741834_1010 (size=4022) 2024-11-13T07:16:48,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741834_1010 (size=4022) 2024-11-13T07:16:48,578 DEBUG [RS:0;2998b885bdef:41365 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/oldWALs 2024-11-13T07:16:48,578 INFO [RS:0;2998b885bdef:41365 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2998b885bdef%2C41365%2C1731482143808:(num 1731482145268) 2024-11-13T07:16:48,578 DEBUG [RS:0;2998b885bdef:41365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:16:48,578 INFO [RS:0;2998b885bdef:41365 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T07:16:48,578 INFO [RS:0;2998b885bdef:41365 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T07:16:48,578 INFO [RS:0;2998b885bdef:41365 {}] hbase.ChoreService(370): Chore service for: regionserver/2998b885bdef:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T07:16:48,578 INFO [RS:0;2998b885bdef:41365 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T07:16:48,578 INFO [RS:0;2998b885bdef:41365 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T07:16:48,578 INFO [regionserver/2998b885bdef:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T07:16:48,579 INFO [RS:0;2998b885bdef:41365 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T07:16:48,579 INFO [RS:0;2998b885bdef:41365 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T07:16:48,579 INFO [RS:0;2998b885bdef:41365 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41365 2024-11-13T07:16:48,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T07:16:48,632 DEBUG [pool-309-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41365-0x101330cc6ab0001, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2998b885bdef,41365,1731482143808 2024-11-13T07:16:48,633 INFO [RS:0;2998b885bdef:41365 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T07:16:48,646 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2998b885bdef,41365,1731482143808] 2024-11-13T07:16:48,664 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2998b885bdef,41365,1731482143808 already deleted, retry=false 2024-11-13T07:16:48,664 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2998b885bdef,41365,1731482143808 expired; onlineServers=1 2024-11-13T07:16:48,750 DEBUG [pool-309-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41365-0x101330cc6ab0001, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T07:16:48,750 DEBUG [pool-309-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41365-0x101330cc6ab0001, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T07:16:48,750 INFO [RS:0;2998b885bdef:41365 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T07:16:48,750 INFO [RS:0;2998b885bdef:41365 {}] regionserver.HRegionServer(1031): Exiting; stopping=2998b885bdef,41365,1731482143808; zookeeper connection closed. 2024-11-13T07:16:48,752 DEBUG [RS:2;2998b885bdef:36543 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T07:16:48,752 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4c29e01d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4c29e01d 2024-11-13T07:16:48,892 INFO [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/hbase/meta/1588230740/.tmp/ns/b2955d14b8ee482bb06daa09100eff24 2024-11-13T07:16:48,937 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/hbase/meta/1588230740/.tmp/table/97e4b8f6ae6a4158a4c1e97939d45902 is 47, key is SPTestTable/table:state/1731482166821/Put/seqid=0 2024-11-13T07:16:48,952 DEBUG [RS:2;2998b885bdef:36543 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T07:16:48,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741866_1042 (size=5214) 2024-11-13T07:16:48,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741866_1042 (size=5214) 2024-11-13T07:16:48,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741866_1042 (size=5214) 2024-11-13T07:16:48,995 INFO [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=86 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/hbase/meta/1588230740/.tmp/table/97e4b8f6ae6a4158a4c1e97939d45902 2024-11-13T07:16:49,019 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/hbase/meta/1588230740/.tmp/info/6e2b203e717c4150abce7792f0de1fc4 as hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/hbase/meta/1588230740/info/6e2b203e717c4150abce7792f0de1fc4 2024-11-13T07:16:49,034 INFO [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/hbase/meta/1588230740/info/6e2b203e717c4150abce7792f0de1fc4, entries=112, sequenceid=35, filesize=19.4 K 2024-11-13T07:16:49,036 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/hbase/meta/1588230740/.tmp/ns/b2955d14b8ee482bb06daa09100eff24 as hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/hbase/meta/1588230740/ns/b2955d14b8ee482bb06daa09100eff24 2024-11-13T07:16:49,054 INFO [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/hbase/meta/1588230740/ns/b2955d14b8ee482bb06daa09100eff24, entries=2, sequenceid=35, filesize=5.0 K 2024-11-13T07:16:49,055 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/hbase/meta/1588230740/.tmp/table/97e4b8f6ae6a4158a4c1e97939d45902 as hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/hbase/meta/1588230740/table/97e4b8f6ae6a4158a4c1e97939d45902 2024-11-13T07:16:49,074 INFO [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/hbase/meta/1588230740/table/97e4b8f6ae6a4158a4c1e97939d45902, entries=2, sequenceid=35, filesize=5.1 K 2024-11-13T07:16:49,077 INFO [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.38 KB/15744, heapSize ~25.51 KB/26120, currentSize=0 B/0 for 1588230740 in 929ms, sequenceid=35, compaction requested=false 2024-11-13T07:16:49,151 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/data/hbase/meta/1588230740/recovered.edits/38.seqid, newMaxSeqId=38, maxSeqId=1 2024-11-13T07:16:49,152 INFO [RS:2;2998b885bdef:36543 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T07:16:49,153 DEBUG [RS:2;2998b885bdef:36543 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-13T07:16:49,153 DEBUG [RS:2;2998b885bdef:36543 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T07:16:49,156 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T07:16:49,156 INFO [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T07:16:49,157 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731482208148Running coprocessor pre-close hooks at 1731482208148Disabling compacts and flushes for region at 1731482208148Disabling writes for close at 1731482208148Obtaining lock to block concurrent updates at 1731482208149 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731482208149Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=15744, getHeapSize=26360, getOffHeapSize=0, getCellsCount=128 at 1731482208149Flushing stores of hbase:meta,,1.1588230740 at 1731482208160 (+11 ms)Flushing 1588230740/info: creating writer at 1731482208160Flushing 1588230740/info: appending metadata at 1731482208212 (+52 ms)Flushing 1588230740/info: closing flushed file at 1731482208212Flushing 1588230740/ns: creating writer at 1731482208348 (+136 ms)Flushing 1588230740/ns: appending metadata at 1731482208396 (+48 ms)Flushing 1588230740/ns: closing flushed file at 1731482208397 (+1 ms)Flushing 1588230740/table: creating writer at 1731482208913 (+516 ms)Flushing 1588230740/table: appending metadata at 1731482208937 (+24 ms)Flushing 1588230740/table: closing flushed file at 1731482208937Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f51be9: reopening flushed file at 1731482209016 (+79 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76fd62d7: reopening flushed file at 1731482209034 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3bbca04a: reopening flushed file at 1731482209054 (+20 ms)Finished flush of dataSize ~15.38 KB/15744, heapSize ~25.51 KB/26120, currentSize=0 B/0 for 1588230740 in 929ms, sequenceid=35, compaction requested=false at 1731482209077 (+23 ms)Writing region close event to WAL at 1731482209118 (+41 ms)Running coprocessor post-close hooks at 1731482209156 (+38 ms)Closed at 1731482209156 2024-11-13T07:16:49,157 DEBUG [RS_CLOSE_META-regionserver/2998b885bdef:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T07:16:49,353 INFO [RS:2;2998b885bdef:36543 {}] regionserver.HRegionServer(976): stopping server 2998b885bdef,36543,1731482143970; all regions closed. 2024-11-13T07:16:49,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741836_1012 (size=18438) 2024-11-13T07:16:49,373 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,36543,1731482143970/2998b885bdef%2C36543%2C1731482143970.meta.1731482155435.meta not finished, retry = 0 2024-11-13T07:16:49,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741836_1012 (size=18438) 2024-11-13T07:16:49,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741836_1012 (size=18438) 2024-11-13T07:16:49,497 DEBUG [RS:2;2998b885bdef:36543 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/oldWALs 2024-11-13T07:16:49,497 INFO [RS:2;2998b885bdef:36543 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2998b885bdef%2C36543%2C1731482143970.meta:.meta(num 1731482155435) 2024-11-13T07:16:49,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741835_1011 (size=3564) 2024-11-13T07:16:49,516 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/WALs/2998b885bdef,36543,1731482143970/2998b885bdef%2C36543%2C1731482143970.1731482145268 not finished, retry = 0 2024-11-13T07:16:49,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741835_1011 (size=3564) 2024-11-13T07:16:49,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741835_1011 (size=3564) 2024-11-13T07:16:49,619 DEBUG [RS:2;2998b885bdef:36543 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/oldWALs 2024-11-13T07:16:49,619 INFO [RS:2;2998b885bdef:36543 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2998b885bdef%2C36543%2C1731482143970:(num 1731482145268) 2024-11-13T07:16:49,619 DEBUG [RS:2;2998b885bdef:36543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T07:16:49,619 INFO [RS:2;2998b885bdef:36543 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T07:16:49,619 INFO [RS:2;2998b885bdef:36543 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T07:16:49,620 INFO [RS:2;2998b885bdef:36543 {}] hbase.ChoreService(370): Chore service for: regionserver/2998b885bdef:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T07:16:49,620 INFO [RS:2;2998b885bdef:36543 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T07:16:49,620 INFO [regionserver/2998b885bdef:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T07:16:49,620 INFO [RS:2;2998b885bdef:36543 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36543 2024-11-13T07:16:49,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T07:16:49,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36543-0x101330cc6ab0003, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2998b885bdef,36543,1731482143970 2024-11-13T07:16:49,845 INFO [RS:2;2998b885bdef:36543 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T07:16:49,878 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2998b885bdef,36543,1731482143970] 2024-11-13T07:16:49,888 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2998b885bdef,36543,1731482143970 already deleted, retry=false 2024-11-13T07:16:49,888 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2998b885bdef,36543,1731482143970 expired; onlineServers=0 2024-11-13T07:16:49,888 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '2998b885bdef,44607,1731482143517' ***** 2024-11-13T07:16:49,888 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T07:16:49,889 INFO [M:0;2998b885bdef:44607 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T07:16:49,889 INFO [M:0;2998b885bdef:44607 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T07:16:49,889 DEBUG [M:0;2998b885bdef:44607 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T07:16:49,889 DEBUG [M:0;2998b885bdef:44607 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T07:16:49,889 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T07:16:49,889 INFO [M:0;2998b885bdef:44607 {}] hbase.ChoreService(370): Chore service for: master/2998b885bdef:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T07:16:49,889 INFO [M:0;2998b885bdef:44607 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T07:16:49,889 DEBUG [master/2998b885bdef:0:becomeActiveMaster-HFileCleaner.small.0-1731482144907 {}] cleaner.HFileCleaner(306): Exit Thread[master/2998b885bdef:0:becomeActiveMaster-HFileCleaner.small.0-1731482144907,5,FailOnTimeoutGroup] 2024-11-13T07:16:49,889 DEBUG [M:0;2998b885bdef:44607 {}] master.HMaster(1795): Stopping service threads 2024-11-13T07:16:49,889 INFO [M:0;2998b885bdef:44607 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T07:16:49,889 INFO [M:0;2998b885bdef:44607 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T07:16:49,889 DEBUG [master/2998b885bdef:0:becomeActiveMaster-HFileCleaner.large.0-1731482144905 {}] cleaner.HFileCleaner(306): Exit Thread[master/2998b885bdef:0:becomeActiveMaster-HFileCleaner.large.0-1731482144905,5,FailOnTimeoutGroup] 2024-11-13T07:16:49,889 ERROR [M:0;2998b885bdef:44607 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1506214772) connection to localhost/127.0.0.1:40989 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:40989,5,PEWorkerGroup] 2024-11-13T07:16:49,889 INFO [M:0;2998b885bdef:44607 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T07:16:49,892 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T07:16:49,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T07:16:49,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T07:16:49,914 DEBUG [M:0;2998b885bdef:44607 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-11-13T07:16:49,914 DEBUG [M:0;2998b885bdef:44607 {}] master.ActiveMasterManager(353): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-11-13T07:16:49,915 INFO [M:0;2998b885bdef:44607 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/.lastflushedseqids 2024-11-13T07:16:49,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741867_1043 (size=650) 2024-11-13T07:16:49,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741867_1043 (size=650) 2024-11-13T07:16:49,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741867_1043 (size=650) 2024-11-13T07:16:49,959 INFO [M:0;2998b885bdef:44607 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T07:16:49,959 INFO [M:0;2998b885bdef:44607 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T07:16:49,959 DEBUG [M:0;2998b885bdef:44607 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T07:16:49,959 INFO [M:0;2998b885bdef:44607 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T07:16:49,960 DEBUG [M:0;2998b885bdef:44607 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T07:16:49,960 DEBUG [M:0;2998b885bdef:44607 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T07:16:49,960 DEBUG [M:0;2998b885bdef:44607 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T07:16:49,960 INFO [M:0;2998b885bdef:44607 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=223.05 KB heapSize=266.23 KB 2024-11-13T07:16:49,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36543-0x101330cc6ab0003, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T07:16:49,981 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36543-0x101330cc6ab0003, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T07:16:49,981 INFO [RS:2;2998b885bdef:36543 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T07:16:49,981 INFO [RS:2;2998b885bdef:36543 {}] regionserver.HRegionServer(1031): Exiting; stopping=2998b885bdef,36543,1731482143970; zookeeper connection closed. 2024-11-13T07:16:49,994 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6d7585f3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6d7585f3 2024-11-13T07:16:49,995 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-13T07:16:50,017 DEBUG [M:0;2998b885bdef:44607 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/78b33deb5b334abda5a4ed61b3d4de12 is 82, key is hbase:meta,,1/info:regioninfo/1731482155520/Put/seqid=0 2024-11-13T07:16:50,036 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=2998b885bdef:44607 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 10 more 2024-11-13T07:16:50,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741868_1044 (size=5672) 2024-11-13T07:16:50,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741868_1044 (size=5672) 2024-11-13T07:16:50,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741868_1044 (size=5672) 2024-11-13T07:16:50,069 INFO [M:0;2998b885bdef:44607 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/78b33deb5b334abda5a4ed61b3d4de12 2024-11-13T07:16:50,119 DEBUG [M:0;2998b885bdef:44607 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2b012e63b0314298a485a445a2f3ccae is 1244, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731482166830/Put/seqid=0 2024-11-13T07:16:50,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741869_1045 (size=9322) 2024-11-13T07:16:50,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741869_1045 (size=9322) 2024-11-13T07:16:50,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741869_1045 (size=9322) 2024-11-13T07:16:50,201 INFO [M:0;2998b885bdef:44607 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=222.32 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2b012e63b0314298a485a445a2f3ccae 2024-11-13T07:16:50,228 INFO [M:0;2998b885bdef:44607 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2b012e63b0314298a485a445a2f3ccae 2024-11-13T07:16:50,267 DEBUG [M:0;2998b885bdef:44607 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8f070ceeb5ff4bbcb29bdd17a277a0f4 is 69, key is 2998b885bdef,36543,1731482143970/rs:state/1731482145058/Put/seqid=0 2024-11-13T07:16:50,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741870_1046 (size=5447) 2024-11-13T07:16:50,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741870_1046 (size=5447) 2024-11-13T07:16:50,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741870_1046 (size=5447) 2024-11-13T07:16:50,312 INFO [M:0;2998b885bdef:44607 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8f070ceeb5ff4bbcb29bdd17a277a0f4 2024-11-13T07:16:50,318 INFO [M:0;2998b885bdef:44607 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8f070ceeb5ff4bbcb29bdd17a277a0f4 2024-11-13T07:16:50,323 DEBUG [M:0;2998b885bdef:44607 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/78b33deb5b334abda5a4ed61b3d4de12 as hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/78b33deb5b334abda5a4ed61b3d4de12 2024-11-13T07:16:50,340 INFO [M:0;2998b885bdef:44607 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/78b33deb5b334abda5a4ed61b3d4de12, entries=8, sequenceid=467, filesize=5.5 K 2024-11-13T07:16:50,341 DEBUG [M:0;2998b885bdef:44607 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2b012e63b0314298a485a445a2f3ccae as hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2b012e63b0314298a485a445a2f3ccae 2024-11-13T07:16:50,370 INFO [M:0;2998b885bdef:44607 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2b012e63b0314298a485a445a2f3ccae 2024-11-13T07:16:50,370 INFO [M:0;2998b885bdef:44607 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2b012e63b0314298a485a445a2f3ccae, entries=54, sequenceid=467, filesize=9.1 K 2024-11-13T07:16:50,376 DEBUG [M:0;2998b885bdef:44607 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8f070ceeb5ff4bbcb29bdd17a277a0f4 as hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8f070ceeb5ff4bbcb29bdd17a277a0f4 2024-11-13T07:16:50,413 INFO [M:0;2998b885bdef:44607 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8f070ceeb5ff4bbcb29bdd17a277a0f4 2024-11-13T07:16:50,413 INFO [M:0;2998b885bdef:44607 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40989/user/jenkins/test-data/b61312b1-07ba-dbb6-53b0-b312b39124ca/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8f070ceeb5ff4bbcb29bdd17a277a0f4, entries=3, sequenceid=467, filesize=5.3 K 2024-11-13T07:16:50,417 INFO [M:0;2998b885bdef:44607 {}] regionserver.HRegion(3140): Finished flush of dataSize ~223.05 KB/228404, heapSize ~265.94 KB/272320, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 457ms, sequenceid=467, compaction requested=false 2024-11-13T07:16:50,437 INFO [M:0;2998b885bdef:44607 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T07:16:50,438 DEBUG [M:0;2998b885bdef:44607 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731482209959Disabling compacts and flushes for region at 1731482209959Disabling writes for close at 1731482209960 (+1 ms)Obtaining lock to block concurrent updates at 1731482209960Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731482209960Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=228404, getHeapSize=272560, getOffHeapSize=0, getCellsCount=570 at 1731482209960Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731482209968 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731482209968Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731482210012 (+44 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731482210012Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731482210087 (+75 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731482210118 (+31 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731482210118Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731482210228 (+110 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731482210265 (+37 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731482210265Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c18887: reopening flushed file at 1731482210319 (+54 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f7fef8e: reopening flushed file at 1731482210340 (+21 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@264e5507: reopening flushed file at 1731482210370 (+30 ms)Finished flush of dataSize ~223.05 KB/228404, heapSize ~265.94 KB/272320, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 457ms, sequenceid=467, compaction requested=false at 1731482210417 (+47 ms)Writing region close event to WAL at 1731482210437 (+20 ms)Closed at 1731482210437 2024-11-13T07:16:50,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46793 is added to blk_1073741830_1006 (size=259590) 2024-11-13T07:16:50,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43743 is added to blk_1073741830_1006 (size=259590) 2024-11-13T07:16:50,454 INFO [M:0;2998b885bdef:44607 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T07:16:50,454 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T07:16:50,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45017 is added to blk_1073741830_1006 (size=259590) 2024-11-13T07:16:50,454 INFO [M:0;2998b885bdef:44607 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44607 2024-11-13T07:16:50,455 INFO [M:0;2998b885bdef:44607 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T07:16:50,615 INFO [M:0;2998b885bdef:44607 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T07:16:50,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T07:16:50,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44607-0x101330cc6ab0000, quorum=127.0.0.1:63253, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T07:16:50,628 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@27eeff62{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T07:16:50,629 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7cfe3ae0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T07:16:50,629 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T07:16:50,629 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@549ff785{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T07:16:50,629 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@74ede052{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/hadoop.log.dir/,STOPPED} 2024-11-13T07:16:50,631 WARN [BP-2139882861-172.17.0.3-1731482139625 heartbeating to localhost/127.0.0.1:40989 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T07:16:50,631 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T07:16:50,631 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T07:16:50,631 WARN [BP-2139882861-172.17.0.3-1731482139625 heartbeating to localhost/127.0.0.1:40989 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2139882861-172.17.0.3-1731482139625 (Datanode Uuid 5dc6eee8-bc83-4b46-861c-9804935efd7d) service to localhost/127.0.0.1:40989 2024-11-13T07:16:50,631 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/cluster_b80e0f79-8e0b-0df0-031b-f8964be40503/data/data5/current/BP-2139882861-172.17.0.3-1731482139625 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T07:16:50,632 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/cluster_b80e0f79-8e0b-0df0-031b-f8964be40503/data/data6/current/BP-2139882861-172.17.0.3-1731482139625 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T07:16:50,632 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T07:16:50,641 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@9234fcc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T07:16:50,642 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@a50cd13{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T07:16:50,642 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T07:16:50,642 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70d5749e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T07:16:50,642 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4de59280{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/hadoop.log.dir/,STOPPED} 2024-11-13T07:16:50,650 WARN [BP-2139882861-172.17.0.3-1731482139625 heartbeating to localhost/127.0.0.1:40989 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T07:16:50,650 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T07:16:50,650 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T07:16:50,650 WARN [BP-2139882861-172.17.0.3-1731482139625 heartbeating to localhost/127.0.0.1:40989 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2139882861-172.17.0.3-1731482139625 (Datanode Uuid 8d67d94d-4516-49da-880d-0d1102fd17e3) service to localhost/127.0.0.1:40989 2024-11-13T07:16:50,651 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/cluster_b80e0f79-8e0b-0df0-031b-f8964be40503/data/data3/current/BP-2139882861-172.17.0.3-1731482139625 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T07:16:50,651 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/cluster_b80e0f79-8e0b-0df0-031b-f8964be40503/data/data4/current/BP-2139882861-172.17.0.3-1731482139625 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T07:16:50,651 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T07:16:50,660 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@bf8095d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T07:16:50,661 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@14ef5eb2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T07:16:50,661 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T07:16:50,661 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@735f3901{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T07:16:50,661 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5aa85305{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/hadoop.log.dir/,STOPPED} 2024-11-13T07:16:50,662 WARN [BP-2139882861-172.17.0.3-1731482139625 heartbeating to localhost/127.0.0.1:40989 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T07:16:50,662 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T07:16:50,663 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T07:16:50,663 WARN [BP-2139882861-172.17.0.3-1731482139625 heartbeating to localhost/127.0.0.1:40989 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2139882861-172.17.0.3-1731482139625 (Datanode Uuid 82436f89-bc06-4064-adfb-71c4761c3e81) service to localhost/127.0.0.1:40989 2024-11-13T07:16:50,663 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/cluster_b80e0f79-8e0b-0df0-031b-f8964be40503/data/data1/current/BP-2139882861-172.17.0.3-1731482139625 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T07:16:50,663 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/cluster_b80e0f79-8e0b-0df0-031b-f8964be40503/data/data2/current/BP-2139882861-172.17.0.3-1731482139625 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T07:16:50,664 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T07:16:50,671 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2ac7a495{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T07:16:50,672 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@29729763{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T07:16:50,672 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T07:16:50,672 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3368b9d6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T07:16:50,673 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7b359d2b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4af5d5a9-06b7-2e4c-c940-7ed80e06c4aa/hadoop.log.dir/,STOPPED} 2024-11-13T07:16:50,682 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T07:16:50,742 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-13T07:16:50,757 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.procedure.TestSnapshotProcedureRSCrashes#testRegionServerCrashWhileVerifyingSnapshot Thread=125 (was 91) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'DataNode' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: HMaster-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40989 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-9-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-10-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-8-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-8-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-12-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-13-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40989 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-11-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40989 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-12-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40989 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-11-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1506214772) connection to localhost/127.0.0.1:40989 from jenkins.hfs.3 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-10-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-9-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:40989 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-10-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1506214772) connection to localhost/127.0.0.1:40989 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1506214772) connection to localhost/127.0.0.1:40989 from jenkins.hfs.5 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-11-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1506214772) connection to localhost/127.0.0.1:40989 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-10-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/2998b885bdef:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:40989 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:40989 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-10-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-13-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-10-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-13-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:40989 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-8-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-12-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-9-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=478 (was 440) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=693 (was 450) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3124 (was 4005)