2024-11-09 21:41:47,571 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-09 21:41:47,583 main DEBUG Took 0.009987 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-09 21:41:47,583 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-09 21:41:47,583 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-09 21:41:47,585 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-09 21:41:47,591 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:41:47,597 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-09 21:41:47,608 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:41:47,609 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:41:47,610 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:41:47,610 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:41:47,610 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:41:47,611 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:41:47,611 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:41:47,612 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:41:47,612 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:41:47,612 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:41:47,613 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:41:47,613 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:41:47,614 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:41:47,614 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:41:47,615 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:41:47,615 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:41:47,615 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:41:47,616 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:41:47,616 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:41:47,616 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:41:47,617 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:41:47,617 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:41:47,618 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:41:47,618 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:41:47,618 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:41:47,618 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-09 21:41:47,620 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:41:47,622 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-09 21:41:47,625 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-09 21:41:47,626 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-09 21:41:47,628 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-09 21:41:47,629 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-09 21:41:47,637 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-09 21:41:47,639 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-09 21:41:47,641 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-09 21:41:47,641 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-09 21:41:47,642 main DEBUG createAppenders(={Console}) 2024-11-09 21:41:47,642 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-11-09 21:41:47,643 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-09 21:41:47,643 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-11-09 21:41:47,643 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-09 21:41:47,644 main DEBUG OutputStream closed 2024-11-09 21:41:47,644 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-09 21:41:47,644 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-09 21:41:47,644 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-11-09 21:41:47,711 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-09 21:41:47,713 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-09 21:41:47,714 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-09 21:41:47,715 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-09 21:41:47,715 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-09 21:41:47,716 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-09 21:41:47,716 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-09 21:41:47,716 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-09 21:41:47,716 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-09 21:41:47,717 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-09 21:41:47,717 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-09 21:41:47,717 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-09 21:41:47,718 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-09 21:41:47,718 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-09 21:41:47,718 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-09 21:41:47,718 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-09 21:41:47,719 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-09 21:41:47,719 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-09 21:41:47,721 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09 21:41:47,722 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-11-09 21:41:47,722 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-09 21:41:47,723 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-11-09T21:41:47,735 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-11-09 21:41:47,737 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-09 21:41:47,737 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09T21:41:47,958 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166 2024-11-09T21:41:47,975 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-09T21:41:47,991 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/cluster_7bb766c4-c195-17b1-f40b-f54a9445f0b2, deleteOnExit=true 2024-11-09T21:41:47,991 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-09T21:41:47,992 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/test.cache.data in system properties and HBase conf 2024-11-09T21:41:47,993 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/hadoop.tmp.dir in system properties and HBase conf 2024-11-09T21:41:47,993 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/hadoop.log.dir in system properties and HBase conf 2024-11-09T21:41:47,994 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-09T21:41:47,995 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-09T21:41:47,995 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-09T21:41:48,071 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-09T21:41:48,154 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T21:41:48,158 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-09T21:41:48,159 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-09T21:41:48,159 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-09T21:41:48,160 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-09T21:41:48,160 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-09T21:41:48,161 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-09T21:41:48,161 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-09T21:41:48,161 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-09T21:41:48,162 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-09T21:41:48,162 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/nfs.dump.dir in system properties and HBase conf 2024-11-09T21:41:48,162 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/java.io.tmpdir in system properties and HBase conf 2024-11-09T21:41:48,163 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-09T21:41:48,163 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-09T21:41:48,164 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-09T21:41:49,104 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-09T21:41:49,182 INFO [Time-limited test {}] log.Log(170): Logging initialized @2270ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-09T21:41:49,248 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T21:41:49,315 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T21:41:49,334 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T21:41:49,335 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T21:41:49,336 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-09T21:41:49,346 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T21:41:49,349 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@13a4dda0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/hadoop.log.dir/,AVAILABLE} 2024-11-09T21:41:49,350 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@38f794dd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T21:41:49,514 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@40b93e99{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/java.io.tmpdir/jetty-localhost-35031-hadoop-hdfs-3_4_1-tests_jar-_-any-6267220503561715640/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-09T21:41:49,523 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4e84709f{HTTP/1.1, (http/1.1)}{localhost:35031} 2024-11-09T21:41:49,523 INFO [Time-limited test {}] server.Server(415): Started @2611ms 2024-11-09T21:41:50,009 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T21:41:50,019 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T21:41:50,020 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T21:41:50,020 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T21:41:50,021 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-09T21:41:50,022 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@79afe1c5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/hadoop.log.dir/,AVAILABLE} 2024-11-09T21:41:50,023 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7f8f7398{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T21:41:50,139 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@429b6cf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/java.io.tmpdir/jetty-localhost-39089-hadoop-hdfs-3_4_1-tests_jar-_-any-11404320571138941179/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T21:41:50,139 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@406d11bc{HTTP/1.1, (http/1.1)}{localhost:39089} 2024-11-09T21:41:50,139 INFO [Time-limited test {}] server.Server(415): Started @3227ms 2024-11-09T21:41:50,191 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-09T21:41:50,985 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/cluster_7bb766c4-c195-17b1-f40b-f54a9445f0b2/data/data2/current/BP-1022936143-172.17.0.2-1731188508680/current, will proceed with Du for space computation calculation, 2024-11-09T21:41:50,985 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/cluster_7bb766c4-c195-17b1-f40b-f54a9445f0b2/data/data1/current/BP-1022936143-172.17.0.2-1731188508680/current, will proceed with Du for space computation calculation, 2024-11-09T21:41:51,012 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-09T21:41:51,061 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2000ef7f9d31b428 with lease ID 0x758bd2debbcac208: Processing first storage report for DS-cfdc8906-16f3-4266-b4d7-9414e9dc84d4 from datanode DatanodeRegistration(127.0.0.1:42537, datanodeUuid=08eec006-7af3-48aa-8495-4ce0dc4f9008, infoPort=40725, infoSecurePort=0, ipcPort=46265, storageInfo=lv=-57;cid=testClusterID;nsid=179446803;c=1731188508680) 2024-11-09T21:41:51,063 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2000ef7f9d31b428 with lease ID 0x758bd2debbcac208: from storage DS-cfdc8906-16f3-4266-b4d7-9414e9dc84d4 node DatanodeRegistration(127.0.0.1:42537, datanodeUuid=08eec006-7af3-48aa-8495-4ce0dc4f9008, infoPort=40725, infoSecurePort=0, ipcPort=46265, storageInfo=lv=-57;cid=testClusterID;nsid=179446803;c=1731188508680), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-09T21:41:51,063 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2000ef7f9d31b428 with lease ID 0x758bd2debbcac208: Processing first storage report for DS-b226950b-6635-4abf-827e-b47b8fcf8909 from datanode DatanodeRegistration(127.0.0.1:42537, datanodeUuid=08eec006-7af3-48aa-8495-4ce0dc4f9008, infoPort=40725, infoSecurePort=0, ipcPort=46265, storageInfo=lv=-57;cid=testClusterID;nsid=179446803;c=1731188508680) 2024-11-09T21:41:51,063 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2000ef7f9d31b428 with lease ID 0x758bd2debbcac208: from storage DS-b226950b-6635-4abf-827e-b47b8fcf8909 node DatanodeRegistration(127.0.0.1:42537, datanodeUuid=08eec006-7af3-48aa-8495-4ce0dc4f9008, infoPort=40725, infoSecurePort=0, ipcPort=46265, storageInfo=lv=-57;cid=testClusterID;nsid=179446803;c=1731188508680), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-09T21:41:51,075 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166 2024-11-09T21:41:51,138 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/cluster_7bb766c4-c195-17b1-f40b-f54a9445f0b2/zookeeper_0, clientPort=62679, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/cluster_7bb766c4-c195-17b1-f40b-f54a9445f0b2/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/cluster_7bb766c4-c195-17b1-f40b-f54a9445f0b2/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-09T21:41:51,150 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62679 2024-11-09T21:41:51,162 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T21:41:51,165 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T21:41:51,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741825_1001 (size=7) 2024-11-09T21:41:51,791 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118 with version=8 2024-11-09T21:41:51,791 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/hbase-staging 2024-11-09T21:41:51,868 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-09T21:41:52,113 INFO [Time-limited test {}] client.ConnectionUtils(128): master/d86ff69d6719:0 server-side Connection retries=9 2024-11-09T21:41:52,123 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T21:41:52,123 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T21:41:52,128 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T21:41:52,128 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T21:41:52,128 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T21:41:52,253 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-09T21:41:52,304 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-09T21:41:52,312 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-09T21:41:52,315 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T21:41:52,336 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 39525 (auto-detected) 2024-11-09T21:41:52,337 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-09T21:41:52,355 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43251 2024-11-09T21:41:52,375 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43251 connecting to ZooKeeper ensemble=127.0.0.1:62679 2024-11-09T21:41:52,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:432510x0, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T21:41:52,468 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43251-0x10122d5e4550000 connected 2024-11-09T21:41:52,538 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T21:41:52,540 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T21:41:52,548 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43251-0x10122d5e4550000, quorum=127.0.0.1:62679, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T21:41:52,552 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118, hbase.cluster.distributed=false 2024-11-09T21:41:52,573 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43251-0x10122d5e4550000, quorum=127.0.0.1:62679, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T21:41:52,578 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43251 2024-11-09T21:41:52,578 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43251 2024-11-09T21:41:52,578 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43251 2024-11-09T21:41:52,583 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43251 2024-11-09T21:41:52,584 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43251 2024-11-09T21:41:52,681 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d86ff69d6719:0 server-side Connection retries=9 2024-11-09T21:41:52,682 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T21:41:52,682 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T21:41:52,683 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T21:41:52,683 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T21:41:52,683 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T21:41:52,686 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-09T21:41:52,689 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T21:41:52,690 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46403 2024-11-09T21:41:52,691 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46403 connecting to ZooKeeper ensemble=127.0.0.1:62679 2024-11-09T21:41:52,693 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T21:41:52,696 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T21:41:52,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:464030x0, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T21:41:52,736 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:464030x0, quorum=127.0.0.1:62679, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T21:41:52,736 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46403-0x10122d5e4550001 connected 2024-11-09T21:41:52,741 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-09T21:41:52,749 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-09T21:41:52,752 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46403-0x10122d5e4550001, quorum=127.0.0.1:62679, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T21:41:52,757 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46403-0x10122d5e4550001, quorum=127.0.0.1:62679, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T21:41:52,758 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46403 2024-11-09T21:41:52,759 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46403 2024-11-09T21:41:52,759 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46403 2024-11-09T21:41:52,760 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46403 2024-11-09T21:41:52,760 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46403 2024-11-09T21:41:52,780 DEBUG [M:0;d86ff69d6719:43251 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d86ff69d6719:43251 2024-11-09T21:41:52,782 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d86ff69d6719,43251,1731188511956 2024-11-09T21:41:52,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46403-0x10122d5e4550001, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T21:41:52,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43251-0x10122d5e4550000, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T21:41:52,805 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43251-0x10122d5e4550000, quorum=127.0.0.1:62679, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d86ff69d6719,43251,1731188511956 2024-11-09T21:41:52,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46403-0x10122d5e4550001, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-09T21:41:52,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43251-0x10122d5e4550000, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:41:52,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46403-0x10122d5e4550001, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:41:52,837 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43251-0x10122d5e4550000, quorum=127.0.0.1:62679, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-09T21:41:52,838 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d86ff69d6719,43251,1731188511956 from backup master directory 2024-11-09T21:41:52,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43251-0x10122d5e4550000, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d86ff69d6719,43251,1731188511956 2024-11-09T21:41:52,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46403-0x10122d5e4550001, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T21:41:52,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43251-0x10122d5e4550000, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T21:41:52,845 WARN [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T21:41:52,845 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d86ff69d6719,43251,1731188511956 2024-11-09T21:41:52,847 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-09T21:41:52,848 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-09T21:41:52,897 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/hbase.id] with ID: 261e20cd-ac9d-4c47-ae34-2a9991c75a0e 2024-11-09T21:41:52,898 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/.tmp/hbase.id 2024-11-09T21:41:52,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741826_1002 (size=42) 2024-11-09T21:41:53,310 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/.tmp/hbase.id]:[hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/hbase.id] 2024-11-09T21:41:53,359 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T21:41:53,363 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-09T21:41:53,381 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-11-09T21:41:53,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43251-0x10122d5e4550000, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:41:53,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46403-0x10122d5e4550001, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:41:53,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741827_1003 (size=196) 2024-11-09T21:41:53,848 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T21:41:53,850 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-09T21:41:53,863 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:41:53,866 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T21:41:53,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741828_1004 (size=1189) 2024-11-09T21:41:54,317 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/MasterData/data/master/store 2024-11-09T21:41:54,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741829_1005 (size=34) 2024-11-09T21:41:54,746 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-09T21:41:54,749 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:41:54,750 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T21:41:54,750 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T21:41:54,751 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T21:41:54,752 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T21:41:54,752 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T21:41:54,753 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T21:41:54,754 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731188514750Disabling compacts and flushes for region at 1731188514750Disabling writes for close at 1731188514752 (+2 ms)Writing region close event to WAL at 1731188514753 (+1 ms)Closed at 1731188514753 2024-11-09T21:41:54,756 WARN [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/MasterData/data/master/store/.initializing 2024-11-09T21:41:54,757 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/MasterData/WALs/d86ff69d6719,43251,1731188511956 2024-11-09T21:41:54,766 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T21:41:54,780 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d86ff69d6719%2C43251%2C1731188511956, suffix=, logDir=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/MasterData/WALs/d86ff69d6719,43251,1731188511956, archiveDir=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/MasterData/oldWALs, maxLogs=10 2024-11-09T21:41:54,808 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/MasterData/WALs/d86ff69d6719,43251,1731188511956/d86ff69d6719%2C43251%2C1731188511956.1731188514786, exclude list is [], retry=0 2024-11-09T21:41:54,823 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42537,DS-cfdc8906-16f3-4266-b4d7-9414e9dc84d4,DISK] 2024-11-09T21:41:54,825 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-09T21:41:54,855 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/MasterData/WALs/d86ff69d6719,43251,1731188511956/d86ff69d6719%2C43251%2C1731188511956.1731188514786 2024-11-09T21:41:54,856 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40725:40725)] 2024-11-09T21:41:54,856 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-09T21:41:54,857 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:41:54,860 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:41:54,860 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:41:54,895 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:41:54,917 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-09T21:41:54,920 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:41:54,922 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:41:54,922 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:41:54,926 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-09T21:41:54,926 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:41:54,927 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:41:54,927 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:41:54,930 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-09T21:41:54,930 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:41:54,931 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:41:54,932 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:41:54,934 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-09T21:41:54,934 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:41:54,935 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:41:54,936 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:41:54,939 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:41:54,940 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:41:54,946 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:41:54,946 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:41:54,950 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T21:41:54,953 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:41:54,958 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:41:54,959 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62434494, jitterRate=-0.06965354084968567}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T21:41:54,966 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731188514871Initializing all the Stores at 1731188514873 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731188514873Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731188514874 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731188514874Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731188514874Cleaning up temporary data from old regions at 1731188514946 (+72 ms)Region opened successfully at 1731188514966 (+20 ms) 2024-11-09T21:41:54,967 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-09T21:41:54,997 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3675267c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d86ff69d6719/172.17.0.2:0 2024-11-09T21:41:55,029 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-09T21:41:55,038 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-09T21:41:55,039 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-09T21:41:55,041 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T21:41:55,042 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-09T21:41:55,046 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-09T21:41:55,047 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-09T21:41:55,074 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-09T21:41:55,083 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43251-0x10122d5e4550000, quorum=127.0.0.1:62679, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-09T21:41:55,127 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-09T21:41:55,131 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-09T21:41:55,133 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43251-0x10122d5e4550000, quorum=127.0.0.1:62679, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-09T21:41:55,144 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-09T21:41:55,148 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-09T21:41:55,155 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43251-0x10122d5e4550000, quorum=127.0.0.1:62679, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-09T21:41:55,160 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-09T21:41:55,162 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43251-0x10122d5e4550000, quorum=127.0.0.1:62679, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-09T21:41:55,169 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-09T21:41:55,187 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43251-0x10122d5e4550000, quorum=127.0.0.1:62679, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-09T21:41:55,194 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-09T21:41:55,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43251-0x10122d5e4550000, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T21:41:55,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43251-0x10122d5e4550000, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:41:55,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46403-0x10122d5e4550001, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T21:41:55,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46403-0x10122d5e4550001, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:41:55,206 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d86ff69d6719,43251,1731188511956, sessionid=0x10122d5e4550000, setting cluster-up flag (Was=false) 2024-11-09T21:41:55,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46403-0x10122d5e4550001, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:41:55,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43251-0x10122d5e4550000, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:41:55,261 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-09T21:41:55,264 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d86ff69d6719,43251,1731188511956 2024-11-09T21:41:55,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43251-0x10122d5e4550000, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:41:55,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46403-0x10122d5e4550001, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:41:55,311 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-09T21:41:55,314 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d86ff69d6719,43251,1731188511956 2024-11-09T21:41:55,321 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-09T21:41:55,367 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(746): ClusterId : 261e20cd-ac9d-4c47-ae34-2a9991c75a0e 2024-11-09T21:41:55,369 DEBUG [RS:0;d86ff69d6719:46403 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-09T21:41:55,387 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T21:41:55,397 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-09T21:41:55,403 DEBUG [RS:0;d86ff69d6719:46403 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-09T21:41:55,403 DEBUG [RS:0;d86ff69d6719:46403 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-09T21:41:55,403 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-09T21:41:55,408 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d86ff69d6719,43251,1731188511956 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-09T21:41:55,412 DEBUG [RS:0;d86ff69d6719:46403 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-09T21:41:55,413 DEBUG [RS:0;d86ff69d6719:46403 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3041154e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d86ff69d6719/172.17.0.2:0 2024-11-09T21:41:55,415 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5 2024-11-09T21:41:55,415 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5 2024-11-09T21:41:55,415 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5 2024-11-09T21:41:55,415 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d86ff69d6719:0, corePoolSize=5, maxPoolSize=5 2024-11-09T21:41:55,415 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d86ff69d6719:0, corePoolSize=10, maxPoolSize=10 2024-11-09T21:41:55,415 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:41:55,416 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d86ff69d6719:0, corePoolSize=2, maxPoolSize=2 2024-11-09T21:41:55,416 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:41:55,423 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T21:41:55,423 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-09T21:41:55,424 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731188545423 2024-11-09T21:41:55,426 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-09T21:41:55,427 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-09T21:41:55,429 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:41:55,429 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-09T21:41:55,430 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d86ff69d6719:46403 2024-11-09T21:41:55,431 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-09T21:41:55,432 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-09T21:41:55,432 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-09T21:41:55,432 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-09T21:41:55,432 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-09T21:41:55,433 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-09T21:41:55,433 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-09T21:41:55,433 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T21:41:55,435 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(2659): reportForDuty to master=d86ff69d6719,43251,1731188511956 with port=46403, startcode=1731188512651 2024-11-09T21:41:55,438 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-09T21:41:55,440 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-09T21:41:55,440 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-09T21:41:55,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741831_1007 (size=1321) 2024-11-09T21:41:55,442 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-09T21:41:55,442 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-09T21:41:55,444 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.large.0-1731188515443,5,FailOnTimeoutGroup] 2024-11-09T21:41:55,445 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.small.0-1731188515444,5,FailOnTimeoutGroup] 2024-11-09T21:41:55,445 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T21:41:55,445 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-09T21:41:55,446 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-09T21:41:55,446 DEBUG [RS:0;d86ff69d6719:46403 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-09T21:41:55,446 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-09T21:41:55,473 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40901, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-09T21:41:55,478 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43251 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d86ff69d6719,46403,1731188512651 2024-11-09T21:41:55,480 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43251 {}] master.ServerManager(517): Registering regionserver=d86ff69d6719,46403,1731188512651 2024-11-09T21:41:55,493 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118 2024-11-09T21:41:55,493 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35923 2024-11-09T21:41:55,493 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-09T21:41:55,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43251-0x10122d5e4550000, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T21:41:55,511 DEBUG [RS:0;d86ff69d6719:46403 {}] zookeeper.ZKUtil(111): regionserver:46403-0x10122d5e4550001, quorum=127.0.0.1:62679, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d86ff69d6719,46403,1731188512651 2024-11-09T21:41:55,512 WARN [RS:0;d86ff69d6719:46403 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T21:41:55,512 INFO [RS:0;d86ff69d6719:46403 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T21:41:55,512 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/WALs/d86ff69d6719,46403,1731188512651 2024-11-09T21:41:55,514 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d86ff69d6719,46403,1731188512651] 2024-11-09T21:41:55,535 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-09T21:41:55,550 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-09T21:41:55,555 INFO [RS:0;d86ff69d6719:46403 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-09T21:41:55,555 INFO [RS:0;d86ff69d6719:46403 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T21:41:55,556 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-09T21:41:55,561 INFO [RS:0;d86ff69d6719:46403 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-09T21:41:55,562 INFO [RS:0;d86ff69d6719:46403 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-09T21:41:55,562 DEBUG [RS:0;d86ff69d6719:46403 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:41:55,562 DEBUG [RS:0;d86ff69d6719:46403 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:41:55,562 DEBUG [RS:0;d86ff69d6719:46403 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:41:55,562 DEBUG [RS:0;d86ff69d6719:46403 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:41:55,563 DEBUG [RS:0;d86ff69d6719:46403 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:41:55,563 DEBUG [RS:0;d86ff69d6719:46403 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d86ff69d6719:0, corePoolSize=2, maxPoolSize=2 2024-11-09T21:41:55,563 DEBUG [RS:0;d86ff69d6719:46403 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:41:55,563 DEBUG [RS:0;d86ff69d6719:46403 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:41:55,563 DEBUG [RS:0;d86ff69d6719:46403 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:41:55,563 DEBUG [RS:0;d86ff69d6719:46403 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:41:55,564 DEBUG [RS:0;d86ff69d6719:46403 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:41:55,564 DEBUG [RS:0;d86ff69d6719:46403 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d86ff69d6719:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:41:55,564 DEBUG [RS:0;d86ff69d6719:46403 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d86ff69d6719:0, corePoolSize=3, maxPoolSize=3 2024-11-09T21:41:55,564 DEBUG [RS:0;d86ff69d6719:46403 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d86ff69d6719:0, corePoolSize=3, maxPoolSize=3 2024-11-09T21:41:55,565 INFO [RS:0;d86ff69d6719:46403 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T21:41:55,565 INFO [RS:0;d86ff69d6719:46403 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T21:41:55,566 INFO [RS:0;d86ff69d6719:46403 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T21:41:55,566 INFO [RS:0;d86ff69d6719:46403 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-09T21:41:55,566 INFO [RS:0;d86ff69d6719:46403 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-09T21:41:55,566 INFO [RS:0;d86ff69d6719:46403 {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,46403,1731188512651-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T21:41:55,585 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-09T21:41:55,587 INFO [RS:0;d86ff69d6719:46403 {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,46403,1731188512651-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T21:41:55,587 INFO [RS:0;d86ff69d6719:46403 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T21:41:55,587 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.Replication(171): d86ff69d6719,46403,1731188512651 started 2024-11-09T21:41:55,609 INFO [RS:0;d86ff69d6719:46403 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T21:41:55,610 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1482): Serving as d86ff69d6719,46403,1731188512651, RpcServer on d86ff69d6719/172.17.0.2:46403, sessionid=0x10122d5e4550001 2024-11-09T21:41:55,610 DEBUG [RS:0;d86ff69d6719:46403 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-09T21:41:55,611 DEBUG [RS:0;d86ff69d6719:46403 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d86ff69d6719,46403,1731188512651 2024-11-09T21:41:55,611 DEBUG [RS:0;d86ff69d6719:46403 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd86ff69d6719,46403,1731188512651' 2024-11-09T21:41:55,611 DEBUG [RS:0;d86ff69d6719:46403 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-09T21:41:55,612 DEBUG [RS:0;d86ff69d6719:46403 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-09T21:41:55,613 DEBUG [RS:0;d86ff69d6719:46403 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-09T21:41:55,613 DEBUG [RS:0;d86ff69d6719:46403 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-09T21:41:55,613 DEBUG [RS:0;d86ff69d6719:46403 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d86ff69d6719,46403,1731188512651 2024-11-09T21:41:55,613 DEBUG [RS:0;d86ff69d6719:46403 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd86ff69d6719,46403,1731188512651' 2024-11-09T21:41:55,613 DEBUG [RS:0;d86ff69d6719:46403 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-09T21:41:55,614 DEBUG [RS:0;d86ff69d6719:46403 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-09T21:41:55,614 DEBUG [RS:0;d86ff69d6719:46403 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-09T21:41:55,614 INFO [RS:0;d86ff69d6719:46403 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-09T21:41:55,614 INFO [RS:0;d86ff69d6719:46403 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-09T21:41:55,848 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-09T21:41:55,849 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118 2024-11-09T21:41:55,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741832_1008 (size=32) 2024-11-09T21:41:55,864 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:41:55,868 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T21:41:55,872 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T21:41:55,872 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:41:55,874 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:41:55,874 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-09T21:41:55,877 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T21:41:55,878 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:41:55,879 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:41:55,879 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T21:41:55,883 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T21:41:55,883 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:41:55,884 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:41:55,884 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T21:41:55,887 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T21:41:55,887 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:41:55,888 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:41:55,888 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T21:41:55,889 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/meta/1588230740 2024-11-09T21:41:55,890 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/meta/1588230740 2024-11-09T21:41:55,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T21:41:55,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T21:41:55,894 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T21:41:55,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T21:41:55,900 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:41:55,901 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58797109, jitterRate=-0.12385480105876923}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T21:41:55,903 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731188515865Initializing all the Stores at 1731188515867 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731188515868 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731188515868Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731188515868Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731188515868Cleaning up temporary data from old regions at 1731188515893 (+25 ms)Region opened successfully at 1731188515903 (+10 ms) 2024-11-09T21:41:55,903 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-09T21:41:55,904 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-09T21:41:55,904 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-09T21:41:55,904 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T21:41:55,904 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T21:41:55,905 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-09T21:41:55,905 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731188515903Disabling compacts and flushes for region at 1731188515903Disabling writes for close at 1731188515904 (+1 ms)Writing region close event to WAL at 1731188515905 (+1 ms)Closed at 1731188515905 2024-11-09T21:41:55,908 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T21:41:55,908 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-09T21:41:55,914 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-09T21:41:55,921 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T21:41:55,923 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-09T21:41:56,075 WARN [d86ff69d6719:43251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:41:57,076 WARN [d86ff69d6719:43251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:41:58,077 WARN [d86ff69d6719:43251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:41:59,079 WARN [d86ff69d6719:43251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:42:00,080 WARN [d86ff69d6719:43251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:42:01,081 WARN [d86ff69d6719:43251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:42:02,082 WARN [d86ff69d6719:43251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:42:02,301 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-09T21:42:02,302 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-09T21:42:03,084 WARN [d86ff69d6719:43251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:42:04,085 WARN [d86ff69d6719:43251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:42:05,086 WARN [d86ff69d6719:43251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:42:06,087 WARN [d86ff69d6719:43251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:42:07,088 WARN [d86ff69d6719:43251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:42:08,089 WARN [d86ff69d6719:43251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:42:09,090 WARN [d86ff69d6719:43251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:42:10,092 WARN [d86ff69d6719:43251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:42:11,093 WARN [d86ff69d6719:43251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:42:12,094 WARN [d86ff69d6719:43251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:42:13,095 WARN [d86ff69d6719:43251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:42:14,096 WARN [d86ff69d6719:43251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:42:15,097 WARN [d86ff69d6719:43251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:42:16,099 WARN [d86ff69d6719:43251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:42:17,100 WARN [d86ff69d6719:43251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:42:18,102 WARN [d86ff69d6719:43251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:42:19,103 WARN [d86ff69d6719:43251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:42:20,104 WARN [d86ff69d6719:43251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:42:21,073 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T21:42:21,105 WARN [d86ff69d6719:43251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:42:22,106 WARN [d86ff69d6719:43251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:42:23,107 WARN [d86ff69d6719:43251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:42:24,108 WARN [d86ff69d6719:43251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:42:25,109 WARN [d86ff69d6719:43251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-09T21:42:25,628 INFO [RS:0;d86ff69d6719:46403 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T21:42:25,632 INFO [RS:0;d86ff69d6719:46403 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d86ff69d6719%2C46403%2C1731188512651, suffix=, logDir=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/WALs/d86ff69d6719,46403,1731188512651, archiveDir=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/oldWALs, maxLogs=32 2024-11-09T21:42:25,649 DEBUG [RS:0;d86ff69d6719:46403 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/WALs/d86ff69d6719,46403,1731188512651/d86ff69d6719%2C46403%2C1731188512651.1731188545635, exclude list is [], retry=0 2024-11-09T21:42:25,653 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42537,DS-cfdc8906-16f3-4266-b4d7-9414e9dc84d4,DISK] 2024-11-09T21:42:25,657 INFO [RS:0;d86ff69d6719:46403 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/WALs/d86ff69d6719,46403,1731188512651/d86ff69d6719%2C46403%2C1731188512651.1731188545635 2024-11-09T21:42:25,657 DEBUG [RS:0;d86ff69d6719:46403 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40725:40725)] 2024-11-09T21:42:25,866 DEBUG [d86ff69d6719:43251 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-09T21:42:25,881 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d86ff69d6719,46403,1731188512651 2024-11-09T21:42:25,886 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d86ff69d6719,46403,1731188512651, state=OPENING 2024-11-09T21:42:25,936 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-09T21:42:25,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46403-0x10122d5e4550001, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:42:25,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43251-0x10122d5e4550000, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:42:25,947 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:42:25,947 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:42:25,950 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T21:42:25,954 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d86ff69d6719,46403,1731188512651}] 2024-11-09T21:42:26,137 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-09T21:42:26,140 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55827, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-09T21:42:26,150 INFO [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-09T21:42:26,150 INFO [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T21:42:26,151 INFO [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-09T21:42:26,154 INFO [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d86ff69d6719%2C46403%2C1731188512651.meta, suffix=.meta, logDir=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/WALs/d86ff69d6719,46403,1731188512651, archiveDir=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/oldWALs, maxLogs=32 2024-11-09T21:42:26,168 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/WALs/d86ff69d6719,46403,1731188512651/d86ff69d6719%2C46403%2C1731188512651.meta.1731188546155.meta, exclude list is [], retry=0 2024-11-09T21:42:26,172 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42537,DS-cfdc8906-16f3-4266-b4d7-9414e9dc84d4,DISK] 2024-11-09T21:42:26,175 INFO [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/WALs/d86ff69d6719,46403,1731188512651/d86ff69d6719%2C46403%2C1731188512651.meta.1731188546155.meta 2024-11-09T21:42:26,175 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40725:40725)] 2024-11-09T21:42:26,176 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-09T21:42:26,177 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-09T21:42:26,179 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-09T21:42:26,183 INFO [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-09T21:42:26,186 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-09T21:42:26,187 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:42:26,187 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-09T21:42:26,187 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-09T21:42:26,190 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T21:42:26,192 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T21:42:26,192 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:42:26,192 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:42:26,193 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-09T21:42:26,194 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T21:42:26,194 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:42:26,195 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:42:26,195 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T21:42:26,196 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T21:42:26,197 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:42:26,197 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:42:26,198 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T21:42:26,199 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T21:42:26,199 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:42:26,200 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:42:26,200 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T21:42:26,201 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/meta/1588230740 2024-11-09T21:42:26,204 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/meta/1588230740 2024-11-09T21:42:26,207 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T21:42:26,207 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T21:42:26,208 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T21:42:26,210 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T21:42:26,212 INFO [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74622405, jitterRate=0.11196048557758331}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T21:42:26,212 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-09T21:42:26,214 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731188546188Writing region info on filesystem at 1731188546188Initializing all the Stores at 1731188546189 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731188546190 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731188546190Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731188546190Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731188546190Cleaning up temporary data from old regions at 1731188546207 (+17 ms)Running coprocessor post-open hooks at 1731188546212 (+5 ms)Region opened successfully at 1731188546214 (+2 ms) 2024-11-09T21:42:26,220 INFO [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731188546130 2024-11-09T21:42:26,230 DEBUG [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-09T21:42:26,231 INFO [RS_OPEN_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-09T21:42:26,232 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d86ff69d6719,46403,1731188512651 2024-11-09T21:42:26,235 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d86ff69d6719,46403,1731188512651, state=OPEN 2024-11-09T21:42:26,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43251-0x10122d5e4550000, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:42:26,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46403-0x10122d5e4550001, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:42:26,288 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:42:26,288 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:42:26,288 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d86ff69d6719,46403,1731188512651 2024-11-09T21:42:26,295 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-09T21:42:26,295 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d86ff69d6719,46403,1731188512651 in 335 msec 2024-11-09T21:42:26,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-09T21:42:26,302 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 30.3830 sec 2024-11-09T21:42:26,304 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T21:42:26,304 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-09T21:42:26,325 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:42:26,326 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d86ff69d6719,46403,1731188512651, seqNum=-1] 2024-11-09T21:42:26,349 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:42:26,351 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36407, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:42:26,373 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 31.0210 sec 2024-11-09T21:42:26,373 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731188546373, completionTime=-1 2024-11-09T21:42:26,375 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-09T21:42:26,376 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-09T21:42:26,398 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-09T21:42:26,398 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731188606398 2024-11-09T21:42:26,398 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731188666398 2024-11-09T21:42:26,398 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 22 msec 2024-11-09T21:42:26,400 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,43251,1731188511956-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T21:42:26,401 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,43251,1731188511956-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T21:42:26,401 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,43251,1731188511956-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T21:42:26,402 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d86ff69d6719:43251, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T21:42:26,402 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-09T21:42:26,403 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-09T21:42:26,408 DEBUG [master/d86ff69d6719:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-09T21:42:26,430 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 33.583sec 2024-11-09T21:42:26,431 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-09T21:42:26,433 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-09T21:42:26,435 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-09T21:42:26,435 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-09T21:42:26,436 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-09T21:42:26,436 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,43251,1731188511956-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T21:42:26,437 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,43251,1731188511956-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-09T21:42:26,445 DEBUG [master/d86ff69d6719:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-09T21:42:26,446 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-09T21:42:26,446 INFO [master/d86ff69d6719:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d86ff69d6719,43251,1731188511956-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T21:42:26,523 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@573d5973, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:42:26,525 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-09T21:42:26,525 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-09T21:42:26,529 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d86ff69d6719,43251,-1 for getting cluster id 2024-11-09T21:42:26,532 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:42:26,539 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '261e20cd-ac9d-4c47-ae34-2a9991c75a0e' 2024-11-09T21:42:26,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:42:26,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "261e20cd-ac9d-4c47-ae34-2a9991c75a0e" 2024-11-09T21:42:26,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@286facfe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:42:26,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d86ff69d6719,43251,-1] 2024-11-09T21:42:26,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:42:26,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:42:26,548 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51338, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:42:26,551 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1928876b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:42:26,551 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:42:26,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d86ff69d6719,46403,1731188512651, seqNum=-1] 2024-11-09T21:42:26,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:42:26,561 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:42:26,578 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d86ff69d6719,43251,1731188511956 2024-11-09T21:42:26,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2246aef7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:42:26,580 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d86ff69d6719,43251,-1 for getting cluster id 2024-11-09T21:42:26,581 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:42:26,582 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '261e20cd-ac9d-4c47-ae34-2a9991c75a0e' 2024-11-09T21:42:26,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:42:26,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "261e20cd-ac9d-4c47-ae34-2a9991c75a0e" 2024-11-09T21:42:26,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@730f9dfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:42:26,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d86ff69d6719,43251,-1] 2024-11-09T21:42:26,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:42:26,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:42:26,586 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51352, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:42:26,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1939ea58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:42:26,600 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=68, ProcessCount=11, AvailableMemoryMB=17910 2024-11-09T21:42:26,606 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T21:42:26,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is d86ff69d6719,43251,1731188511956 2024-11-09T21:42:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@539c366a 2024-11-09T21:42:26,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T21:42:26,639 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T21:42:26,646 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T21:42:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-11-09T21:42:26,662 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T21:42:26,664 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:42:26,664 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-11-09T21:42:26,667 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T21:42:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T21:42:26,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741835_1011 (size=428) 2024-11-09T21:42:27,084 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 08ca2f06be250b50389ea4bf06a0b92b, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1731188546641.08ca2f06be250b50389ea4bf06a0b92b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118 2024-11-09T21:42:27,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741836_1012 (size=79) 2024-11-09T21:42:27,499 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1731188546641.08ca2f06be250b50389ea4bf06a0b92b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:42:27,500 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing 08ca2f06be250b50389ea4bf06a0b92b, disabling compactions & flushes 2024-11-09T21:42:27,500 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1731188546641.08ca2f06be250b50389ea4bf06a0b92b. 2024-11-09T21:42:27,500 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731188546641.08ca2f06be250b50389ea4bf06a0b92b. 2024-11-09T21:42:27,501 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731188546641.08ca2f06be250b50389ea4bf06a0b92b. after waiting 0 ms 2024-11-09T21:42:27,501 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1731188546641.08ca2f06be250b50389ea4bf06a0b92b. 2024-11-09T21:42:27,501 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1731188546641.08ca2f06be250b50389ea4bf06a0b92b. 2024-11-09T21:42:27,501 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 08ca2f06be250b50389ea4bf06a0b92b: Waiting for close lock at 1731188547500Disabling compacts and flushes for region at 1731188547500Disabling writes for close at 1731188547501 (+1 ms)Writing region close event to WAL at 1731188547501Closed at 1731188547501 2024-11-09T21:42:27,506 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T21:42:27,510 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1731188546641.08ca2f06be250b50389ea4bf06a0b92b.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731188547506"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731188547506"}]},"ts":"1731188547506"} 2024-11-09T21:42:27,514 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T21:42:27,516 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T21:42:27,519 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731188547516"}]},"ts":"1731188547516"} 2024-11-09T21:42:27,523 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-11-09T21:42:27,525 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=08ca2f06be250b50389ea4bf06a0b92b, ASSIGN}] 2024-11-09T21:42:27,528 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=08ca2f06be250b50389ea4bf06a0b92b, ASSIGN 2024-11-09T21:42:27,530 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=08ca2f06be250b50389ea4bf06a0b92b, ASSIGN; state=OFFLINE, location=d86ff69d6719,46403,1731188512651; forceNewPlan=false, retain=false 2024-11-09T21:42:27,682 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=08ca2f06be250b50389ea4bf06a0b92b, regionState=OPENING, regionLocation=d86ff69d6719,46403,1731188512651 2024-11-09T21:42:27,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T21:42:27,689 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=08ca2f06be250b50389ea4bf06a0b92b, ASSIGN because future has completed 2024-11-09T21:42:27,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08ca2f06be250b50389ea4bf06a0b92b, server=d86ff69d6719,46403,1731188512651}] 2024-11-09T21:42:27,857 INFO [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1731188546641.08ca2f06be250b50389ea4bf06a0b92b. 2024-11-09T21:42:27,857 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 08ca2f06be250b50389ea4bf06a0b92b, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1731188546641.08ca2f06be250b50389ea4bf06a0b92b.', STARTKEY => '', ENDKEY => ''} 2024-11-09T21:42:27,857 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync 08ca2f06be250b50389ea4bf06a0b92b 2024-11-09T21:42:27,858 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1731188546641.08ca2f06be250b50389ea4bf06a0b92b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:42:27,858 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 08ca2f06be250b50389ea4bf06a0b92b 2024-11-09T21:42:27,858 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 08ca2f06be250b50389ea4bf06a0b92b 2024-11-09T21:42:27,860 INFO [StoreOpener-08ca2f06be250b50389ea4bf06a0b92b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 08ca2f06be250b50389ea4bf06a0b92b 2024-11-09T21:42:27,863 INFO [StoreOpener-08ca2f06be250b50389ea4bf06a0b92b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08ca2f06be250b50389ea4bf06a0b92b columnFamilyName testFamily 2024-11-09T21:42:27,863 DEBUG [StoreOpener-08ca2f06be250b50389ea4bf06a0b92b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:42:27,864 INFO [StoreOpener-08ca2f06be250b50389ea4bf06a0b92b-1 {}] regionserver.HStore(327): Store=08ca2f06be250b50389ea4bf06a0b92b/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:42:27,865 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 08ca2f06be250b50389ea4bf06a0b92b 2024-11-09T21:42:27,867 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testRetryOutOfOrderScannerNextExceptionAsync/08ca2f06be250b50389ea4bf06a0b92b 2024-11-09T21:42:27,867 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testRetryOutOfOrderScannerNextExceptionAsync/08ca2f06be250b50389ea4bf06a0b92b 2024-11-09T21:42:27,868 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 08ca2f06be250b50389ea4bf06a0b92b 2024-11-09T21:42:27,868 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 08ca2f06be250b50389ea4bf06a0b92b 2024-11-09T21:42:27,871 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 08ca2f06be250b50389ea4bf06a0b92b 2024-11-09T21:42:27,874 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testRetryOutOfOrderScannerNextExceptionAsync/08ca2f06be250b50389ea4bf06a0b92b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:42:27,876 INFO [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 08ca2f06be250b50389ea4bf06a0b92b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58814663, jitterRate=-0.12359322607517242}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:42:27,876 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 08ca2f06be250b50389ea4bf06a0b92b 2024-11-09T21:42:27,877 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 08ca2f06be250b50389ea4bf06a0b92b: Running coprocessor pre-open hook at 1731188547858Writing region info on filesystem at 1731188547858Initializing all the Stores at 1731188547860 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731188547860Cleaning up temporary data from old regions at 1731188547868 (+8 ms)Running coprocessor post-open hooks at 1731188547876 (+8 ms)Region opened successfully at 1731188547877 (+1 ms) 2024-11-09T21:42:27,879 INFO [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1731188546641.08ca2f06be250b50389ea4bf06a0b92b., pid=6, masterSystemTime=1731188547846 2024-11-09T21:42:27,882 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1731188546641.08ca2f06be250b50389ea4bf06a0b92b. 2024-11-09T21:42:27,883 INFO [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1731188546641.08ca2f06be250b50389ea4bf06a0b92b. 2024-11-09T21:42:27,884 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=08ca2f06be250b50389ea4bf06a0b92b, regionState=OPEN, openSeqNum=2, regionLocation=d86ff69d6719,46403,1731188512651 2024-11-09T21:42:27,888 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 08ca2f06be250b50389ea4bf06a0b92b, server=d86ff69d6719,46403,1731188512651 because future has completed 2024-11-09T21:42:27,895 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-09T21:42:27,895 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 08ca2f06be250b50389ea4bf06a0b92b, server=d86ff69d6719,46403,1731188512651 in 199 msec 2024-11-09T21:42:27,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-09T21:42:27,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=08ca2f06be250b50389ea4bf06a0b92b, ASSIGN in 370 msec 2024-11-09T21:42:27,900 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T21:42:27,900 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731188547900"}]},"ts":"1731188547900"} 2024-11-09T21:42:27,903 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-11-09T21:42:27,905 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T21:42:27,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 1.2550 sec 2024-11-09T21:42:29,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T21:42:29,701 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-11-09T21:42:29,702 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-11-09T21:42:29,707 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T21:42:29,711 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-11-09T21:42:29,712 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T21:42:29,712 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-11-09T21:42:29,714 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:42:29,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d86ff69d6719,46403,1731188512651, seqNum=-1] 2024-11-09T21:42:29,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:42:29,719 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56192, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:42:29,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1731188546641.08ca2f06be250b50389ea4bf06a0b92b., hostname=d86ff69d6719,46403,1731188512651, seqNum=2] 2024-11-09T21:42:29,740 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-09T21:42:29,743 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-11-09T21:42:29,748 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-09T21:42:29,753 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-09T21:42:29,755 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-09T21:42:29,756 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46403 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-8853385579539202040 2024-11-09T21:42:29,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46403 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:56192 deadline: 1731188552754, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T21:42:29,780 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-09T21:42:29,782 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-09T21:42:29,783 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-09T21:42:29,785 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-09T21:42:29,786 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46403 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-8853385579539202038 2024-11-09T21:42:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46403 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:56192 deadline: 1731188552785, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T21:42:29,790 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-09T21:42:29,791 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46403 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-8853385579539202037 2024-11-09T21:42:29,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46403 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:56192 deadline: 1731188552790, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T21:42:29,795 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-09T21:42:29,796 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46403 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-8853385579539202036 2024-11-09T21:42:29,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46403 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:56192 deadline: 1731188552795, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T21:42:29,800 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-09T21:42:29,809 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=220 (was 214) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1838668448_22 at /127.0.0.1:49626 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=68 (was 68), ProcessCount=11 (was 11), AvailableMemoryMB=17865 (was 17910) 2024-11-09T21:42:29,817 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=220, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=68, ProcessCount=11, AvailableMemoryMB=17864 2024-11-09T21:42:29,820 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43251 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T21:42:29,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43251 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-11-09T21:42:29,824 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T21:42:29,825 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:42:29,825 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43251 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-11-09T21:42:29,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T21:42:29,827 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T21:42:29,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741837_1013 (size=414) 2024-11-09T21:42:30,241 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 20d9071050465d83b5f5ea0aba708a01, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1731188549820.20d9071050465d83b5f5ea0aba708a01.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118 2024-11-09T21:42:30,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741838_1014 (size=65) 2024-11-09T21:42:30,654 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1731188549820.20d9071050465d83b5f5ea0aba708a01.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:42:30,654 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 20d9071050465d83b5f5ea0aba708a01, disabling compactions & flushes 2024-11-09T21:42:30,655 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1731188549820.20d9071050465d83b5f5ea0aba708a01. 2024-11-09T21:42:30,655 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731188549820.20d9071050465d83b5f5ea0aba708a01. 2024-11-09T21:42:30,655 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731188549820.20d9071050465d83b5f5ea0aba708a01. after waiting 0 ms 2024-11-09T21:42:30,655 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1731188549820.20d9071050465d83b5f5ea0aba708a01. 2024-11-09T21:42:30,655 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1731188549820.20d9071050465d83b5f5ea0aba708a01. 2024-11-09T21:42:30,655 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 20d9071050465d83b5f5ea0aba708a01: Waiting for close lock at 1731188550654Disabling compacts and flushes for region at 1731188550654Disabling writes for close at 1731188550655 (+1 ms)Writing region close event to WAL at 1731188550655Closed at 1731188550655 2024-11-09T21:42:30,657 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T21:42:30,658 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1731188549820.20d9071050465d83b5f5ea0aba708a01.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1731188550657"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731188550657"}]},"ts":"1731188550657"} 2024-11-09T21:42:30,662 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T21:42:30,665 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T21:42:30,665 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731188550665"}]},"ts":"1731188550665"} 2024-11-09T21:42:30,668 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-09T21:42:30,668 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=20d9071050465d83b5f5ea0aba708a01, ASSIGN}] 2024-11-09T21:42:30,670 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=20d9071050465d83b5f5ea0aba708a01, ASSIGN 2024-11-09T21:42:30,671 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=20d9071050465d83b5f5ea0aba708a01, ASSIGN; state=OFFLINE, location=d86ff69d6719,46403,1731188512651; forceNewPlan=false, retain=false 2024-11-09T21:42:30,823 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=20d9071050465d83b5f5ea0aba708a01, regionState=OPENING, regionLocation=d86ff69d6719,46403,1731188512651 2024-11-09T21:42:30,833 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=20d9071050465d83b5f5ea0aba708a01, ASSIGN because future has completed 2024-11-09T21:42:30,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 20d9071050465d83b5f5ea0aba708a01, server=d86ff69d6719,46403,1731188512651}] 2024-11-09T21:42:30,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T21:42:31,001 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1731188549820.20d9071050465d83b5f5ea0aba708a01. 2024-11-09T21:42:31,002 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 20d9071050465d83b5f5ea0aba708a01, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1731188549820.20d9071050465d83b5f5ea0aba708a01.', STARTKEY => '', ENDKEY => ''} 2024-11-09T21:42:31,002 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner 20d9071050465d83b5f5ea0aba708a01 2024-11-09T21:42:31,002 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1731188549820.20d9071050465d83b5f5ea0aba708a01.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:42:31,003 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 20d9071050465d83b5f5ea0aba708a01 2024-11-09T21:42:31,003 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 20d9071050465d83b5f5ea0aba708a01 2024-11-09T21:42:31,005 INFO [StoreOpener-20d9071050465d83b5f5ea0aba708a01-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 20d9071050465d83b5f5ea0aba708a01 2024-11-09T21:42:31,008 INFO [StoreOpener-20d9071050465d83b5f5ea0aba708a01-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 20d9071050465d83b5f5ea0aba708a01 columnFamilyName testFamily 2024-11-09T21:42:31,008 DEBUG [StoreOpener-20d9071050465d83b5f5ea0aba708a01-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:42:31,009 INFO [StoreOpener-20d9071050465d83b5f5ea0aba708a01-1 {}] regionserver.HStore(327): Store=20d9071050465d83b5f5ea0aba708a01/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:42:31,009 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 20d9071050465d83b5f5ea0aba708a01 2024-11-09T21:42:31,010 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnOpenScanner/20d9071050465d83b5f5ea0aba708a01 2024-11-09T21:42:31,011 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnOpenScanner/20d9071050465d83b5f5ea0aba708a01 2024-11-09T21:42:31,012 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 20d9071050465d83b5f5ea0aba708a01 2024-11-09T21:42:31,012 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 20d9071050465d83b5f5ea0aba708a01 2024-11-09T21:42:31,015 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 20d9071050465d83b5f5ea0aba708a01 2024-11-09T21:42:31,019 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnOpenScanner/20d9071050465d83b5f5ea0aba708a01/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:42:31,020 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened 20d9071050465d83b5f5ea0aba708a01; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64389099, jitterRate=-0.04052765667438507}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:42:31,021 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 20d9071050465d83b5f5ea0aba708a01 2024-11-09T21:42:31,022 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 20d9071050465d83b5f5ea0aba708a01: Running coprocessor pre-open hook at 1731188551003Writing region info on filesystem at 1731188551003Initializing all the Stores at 1731188551005 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731188551005Cleaning up temporary data from old regions at 1731188551012 (+7 ms)Running coprocessor post-open hooks at 1731188551021 (+9 ms)Region opened successfully at 1731188551022 (+1 ms) 2024-11-09T21:42:31,023 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1731188549820.20d9071050465d83b5f5ea0aba708a01., pid=9, masterSystemTime=1731188550989 2024-11-09T21:42:31,027 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1731188549820.20d9071050465d83b5f5ea0aba708a01. 2024-11-09T21:42:31,027 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1731188549820.20d9071050465d83b5f5ea0aba708a01. 2024-11-09T21:42:31,028 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=20d9071050465d83b5f5ea0aba708a01, regionState=OPEN, openSeqNum=2, regionLocation=d86ff69d6719,46403,1731188512651 2024-11-09T21:42:31,031 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 20d9071050465d83b5f5ea0aba708a01, server=d86ff69d6719,46403,1731188512651 because future has completed 2024-11-09T21:42:31,037 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-09T21:42:31,037 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 20d9071050465d83b5f5ea0aba708a01, server=d86ff69d6719,46403,1731188512651 in 199 msec 2024-11-09T21:42:31,042 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-09T21:42:31,042 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=20d9071050465d83b5f5ea0aba708a01, ASSIGN in 369 msec 2024-11-09T21:42:31,044 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T21:42:31,044 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731188551044"}]},"ts":"1731188551044"} 2024-11-09T21:42:31,047 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-09T21:42:31,049 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T21:42:31,053 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2290 sec 2024-11-09T21:42:31,724 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T21:42:31,753 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-11-09T21:42:31,754 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-09T21:42:31,755 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-11-09T21:42:32,301 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-09T21:42:32,301 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-09T21:42:32,305 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T21:42:32,305 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-09T21:42:32,306 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-09T21:42:32,306 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-11-09T21:42:32,307 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-09T21:42:32,308 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-09T21:42:32,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T21:42:32,849 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-11-09T21:42:32,849 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-09T21:42:32,849 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T21:42:32,857 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-09T21:42:32,858 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T21:42:32,858 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-11-09T21:42:32,862 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1731188549820.20d9071050465d83b5f5ea0aba708a01., hostname=d86ff69d6719,46403,1731188512651, seqNum=2] 2024-11-09T21:42:32,873 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-09T21:42:32,873 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-09T21:42:32,877 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-09T21:42:32,879 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46403 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-09T21:42:38,898 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1731188549820.20d9071050465d83b5f5ea0aba708a01., hostname=d86ff69d6719,46403,1731188512651, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1731188549820.20d9071050465d83b5f5ea0aba708a01., hostname=d86ff69d6719,46403,1731188512651, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d86ff69d6719:46403 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6015ms, rpcTimeout=5999ms 2024-11-09T21:42:38,899 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1731188549820.20d9071050465d83b5f5ea0aba708a01., hostname=d86ff69d6719,46403,1731188512651, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6015ms, rpcTimeout=5999ms 2024-11-09T21:42:38,899 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1731188549820.20d9071050465d83b5f5ea0aba708a01., hostname=d86ff69d6719,46403,1731188512651, seqNum=2 from cache 2024-11-09T21:42:38,908 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=223 (was 220) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=72 (was 68) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=17850 (was 17864) 2024-11-09T21:42:38,916 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=72, ProcessCount=11, AvailableMemoryMB=17850 2024-11-09T21:42:38,918 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43251 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T21:42:38,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43251 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-11-09T21:42:38,922 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T21:42:38,923 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:42:38,923 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43251 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-11-09T21:42:38,924 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T21:42:38,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-09T21:42:38,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741839_1015 (size=412) 2024-11-09T21:42:39,342 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0bb8274b2699870e24bf8df1a279f881, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1731188558918.0bb8274b2699870e24bf8df1a279f881.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118 2024-11-09T21:42:39,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741840_1016 (size=63) 2024-11-09T21:42:39,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 89 bytes 2024-11-09T21:42:39,759 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1731188558918.0bb8274b2699870e24bf8df1a279f881.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:42:39,760 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 0bb8274b2699870e24bf8df1a279f881, disabling compactions & flushes 2024-11-09T21:42:39,760 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1731188558918.0bb8274b2699870e24bf8df1a279f881. 2024-11-09T21:42:39,760 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731188558918.0bb8274b2699870e24bf8df1a279f881. 2024-11-09T21:42:39,760 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731188558918.0bb8274b2699870e24bf8df1a279f881. after waiting 0 ms 2024-11-09T21:42:39,760 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1731188558918.0bb8274b2699870e24bf8df1a279f881. 2024-11-09T21:42:39,761 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1731188558918.0bb8274b2699870e24bf8df1a279f881. 2024-11-09T21:42:39,761 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0bb8274b2699870e24bf8df1a279f881: Waiting for close lock at 1731188559759Disabling compacts and flushes for region at 1731188559759Disabling writes for close at 1731188559760 (+1 ms)Writing region close event to WAL at 1731188559760Closed at 1731188559760 2024-11-09T21:42:39,766 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T21:42:39,767 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1731188558918.0bb8274b2699870e24bf8df1a279f881.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1731188559766"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731188559766"}]},"ts":"1731188559766"} 2024-11-09T21:42:39,770 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T21:42:39,772 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T21:42:39,772 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731188559772"}]},"ts":"1731188559772"} 2024-11-09T21:42:39,775 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-09T21:42:39,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=0bb8274b2699870e24bf8df1a279f881, ASSIGN}] 2024-11-09T21:42:39,777 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=0bb8274b2699870e24bf8df1a279f881, ASSIGN 2024-11-09T21:42:39,779 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=0bb8274b2699870e24bf8df1a279f881, ASSIGN; state=OFFLINE, location=d86ff69d6719,46403,1731188512651; forceNewPlan=false, retain=false 2024-11-09T21:42:39,930 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=0bb8274b2699870e24bf8df1a279f881, regionState=OPENING, regionLocation=d86ff69d6719,46403,1731188512651 2024-11-09T21:42:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-09T21:42:39,941 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=0bb8274b2699870e24bf8df1a279f881, ASSIGN because future has completed 2024-11-09T21:42:39,942 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0bb8274b2699870e24bf8df1a279f881, server=d86ff69d6719,46403,1731188512651}] 2024-11-09T21:42:40,109 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1731188558918.0bb8274b2699870e24bf8df1a279f881. 2024-11-09T21:42:40,110 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 0bb8274b2699870e24bf8df1a279f881, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1731188558918.0bb8274b2699870e24bf8df1a279f881.', STARTKEY => '', ENDKEY => ''} 2024-11-09T21:42:40,110 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync 0bb8274b2699870e24bf8df1a279f881 2024-11-09T21:42:40,110 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1731188558918.0bb8274b2699870e24bf8df1a279f881.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:42:40,111 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 0bb8274b2699870e24bf8df1a279f881 2024-11-09T21:42:40,111 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 0bb8274b2699870e24bf8df1a279f881 2024-11-09T21:42:40,113 INFO [StoreOpener-0bb8274b2699870e24bf8df1a279f881-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 0bb8274b2699870e24bf8df1a279f881 2024-11-09T21:42:40,116 INFO [StoreOpener-0bb8274b2699870e24bf8df1a279f881-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0bb8274b2699870e24bf8df1a279f881 columnFamilyName testFamily 2024-11-09T21:42:40,116 DEBUG [StoreOpener-0bb8274b2699870e24bf8df1a279f881-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:42:40,117 INFO [StoreOpener-0bb8274b2699870e24bf8df1a279f881-1 {}] regionserver.HStore(327): Store=0bb8274b2699870e24bf8df1a279f881/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:42:40,117 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 0bb8274b2699870e24bf8df1a279f881 2024-11-09T21:42:40,119 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnNextAsync/0bb8274b2699870e24bf8df1a279f881 2024-11-09T21:42:40,120 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnNextAsync/0bb8274b2699870e24bf8df1a279f881 2024-11-09T21:42:40,120 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 0bb8274b2699870e24bf8df1a279f881 2024-11-09T21:42:40,120 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 0bb8274b2699870e24bf8df1a279f881 2024-11-09T21:42:40,123 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 0bb8274b2699870e24bf8df1a279f881 2024-11-09T21:42:40,127 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnNextAsync/0bb8274b2699870e24bf8df1a279f881/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:42:40,127 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened 0bb8274b2699870e24bf8df1a279f881; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63754080, jitterRate=-0.049990177154541016}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:42:40,128 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0bb8274b2699870e24bf8df1a279f881 2024-11-09T21:42:40,128 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 0bb8274b2699870e24bf8df1a279f881: Running coprocessor pre-open hook at 1731188560111Writing region info on filesystem at 1731188560111Initializing all the Stores at 1731188560113 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731188560113Cleaning up temporary data from old regions at 1731188560120 (+7 ms)Running coprocessor post-open hooks at 1731188560128 (+8 ms)Region opened successfully at 1731188560128 2024-11-09T21:42:40,129 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1731188558918.0bb8274b2699870e24bf8df1a279f881., pid=12, masterSystemTime=1731188560097 2024-11-09T21:42:40,132 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1731188558918.0bb8274b2699870e24bf8df1a279f881. 2024-11-09T21:42:40,132 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1731188558918.0bb8274b2699870e24bf8df1a279f881. 2024-11-09T21:42:40,133 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=0bb8274b2699870e24bf8df1a279f881, regionState=OPEN, openSeqNum=2, regionLocation=d86ff69d6719,46403,1731188512651 2024-11-09T21:42:40,136 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0bb8274b2699870e24bf8df1a279f881, server=d86ff69d6719,46403,1731188512651 because future has completed 2024-11-09T21:42:40,141 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-09T21:42:40,141 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 0bb8274b2699870e24bf8df1a279f881, server=d86ff69d6719,46403,1731188512651 in 196 msec 2024-11-09T21:42:40,144 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-09T21:42:40,144 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=0bb8274b2699870e24bf8df1a279f881, ASSIGN in 366 msec 2024-11-09T21:42:40,145 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T21:42:40,145 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731188560145"}]},"ts":"1731188560145"} 2024-11-09T21:42:40,148 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-09T21:42:40,150 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T21:42:40,152 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 1.2310 sec 2024-11-09T21:42:41,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-09T21:42:41,968 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-11-09T21:42:41,968 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-09T21:42:41,968 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T21:42:41,978 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-09T21:42:41,978 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T21:42:41,978 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-11-09T21:42:41,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1731188558918.0bb8274b2699870e24bf8df1a279f881., hostname=d86ff69d6719,46403,1731188512651, seqNum=2] 2024-11-09T21:42:41,993 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-09T21:42:41,993 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-09T21:42:41,997 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T21:42:41,999 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-09T21:42:42,000 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-09T21:42:42,001 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46403 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-09T21:42:42,301 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-09T21:42:42,301 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-09T21:42:43,758 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-11-09T21:42:45,642 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T21:42:48,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46403 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8853385579539202027' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-09T21:42:48,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46403 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:56192 deadline: 1731188574017, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8853385579539202027' does not exist 2024-11-09T21:42:48,020 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to d86ff69d6719,46403,1731188512651 for closing scanner id = -8853385579539202027 for 0bb8274b2699870e24bf8df1a279f881 of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8853385579539202027' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:42:48,023 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=225 (was 223) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=68 (was 72), ProcessCount=11 (was 11), AvailableMemoryMB=17842 (was 17850) 2024-11-09T21:42:48,030 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=225, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=68, ProcessCount=11, AvailableMemoryMB=17842 2024-11-09T21:42:48,032 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T21:42:48,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-11-09T21:42:48,036 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T21:42:48,036 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:42:48,036 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-11-09T21:42:48,038 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T21:42:48,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T21:42:48,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741841_1017 (size=423) 2024-11-09T21:42:48,456 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d2c3f456b99d4783fd5f2aeb363ce6d7, NAME => 'testRetryOutOfOrderScannerNextException,,1731188568032.d2c3f456b99d4783fd5f2aeb363ce6d7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118 2024-11-09T21:42:48,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 83 bytes 2024-11-09T21:42:48,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741842_1018 (size=74) 2024-11-09T21:42:48,909 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1731188568032.d2c3f456b99d4783fd5f2aeb363ce6d7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:42:48,910 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing d2c3f456b99d4783fd5f2aeb363ce6d7, disabling compactions & flushes 2024-11-09T21:42:48,910 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1731188568032.d2c3f456b99d4783fd5f2aeb363ce6d7. 2024-11-09T21:42:48,910 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1731188568032.d2c3f456b99d4783fd5f2aeb363ce6d7. 2024-11-09T21:42:48,910 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1731188568032.d2c3f456b99d4783fd5f2aeb363ce6d7. after waiting 0 ms 2024-11-09T21:42:48,911 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1731188568032.d2c3f456b99d4783fd5f2aeb363ce6d7. 2024-11-09T21:42:48,911 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1731188568032.d2c3f456b99d4783fd5f2aeb363ce6d7. 2024-11-09T21:42:48,911 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for d2c3f456b99d4783fd5f2aeb363ce6d7: Waiting for close lock at 1731188568910Disabling compacts and flushes for region at 1731188568910Disabling writes for close at 1731188568911 (+1 ms)Writing region close event to WAL at 1731188568911Closed at 1731188568911 2024-11-09T21:42:48,916 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T21:42:48,917 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1731188568032.d2c3f456b99d4783fd5f2aeb363ce6d7.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1731188568916"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731188568916"}]},"ts":"1731188568916"} 2024-11-09T21:42:48,921 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T21:42:48,924 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T21:42:48,924 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731188568924"}]},"ts":"1731188568924"} 2024-11-09T21:42:48,927 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-11-09T21:42:48,927 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=d2c3f456b99d4783fd5f2aeb363ce6d7, ASSIGN}] 2024-11-09T21:42:48,929 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=d2c3f456b99d4783fd5f2aeb363ce6d7, ASSIGN 2024-11-09T21:42:48,930 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=d2c3f456b99d4783fd5f2aeb363ce6d7, ASSIGN; state=OFFLINE, location=d86ff69d6719,46403,1731188512651; forceNewPlan=false, retain=false 2024-11-09T21:42:49,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T21:42:49,082 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=d2c3f456b99d4783fd5f2aeb363ce6d7, regionState=OPENING, regionLocation=d86ff69d6719,46403,1731188512651 2024-11-09T21:42:49,089 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=d2c3f456b99d4783fd5f2aeb363ce6d7, ASSIGN because future has completed 2024-11-09T21:42:49,091 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2c3f456b99d4783fd5f2aeb363ce6d7, server=d86ff69d6719,46403,1731188512651}] 2024-11-09T21:42:49,252 INFO [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1731188568032.d2c3f456b99d4783fd5f2aeb363ce6d7. 2024-11-09T21:42:49,253 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => d2c3f456b99d4783fd5f2aeb363ce6d7, NAME => 'testRetryOutOfOrderScannerNextException,,1731188568032.d2c3f456b99d4783fd5f2aeb363ce6d7.', STARTKEY => '', ENDKEY => ''} 2024-11-09T21:42:49,253 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException d2c3f456b99d4783fd5f2aeb363ce6d7 2024-11-09T21:42:49,253 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1731188568032.d2c3f456b99d4783fd5f2aeb363ce6d7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:42:49,253 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for d2c3f456b99d4783fd5f2aeb363ce6d7 2024-11-09T21:42:49,254 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for d2c3f456b99d4783fd5f2aeb363ce6d7 2024-11-09T21:42:49,256 INFO [StoreOpener-d2c3f456b99d4783fd5f2aeb363ce6d7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region d2c3f456b99d4783fd5f2aeb363ce6d7 2024-11-09T21:42:49,259 INFO [StoreOpener-d2c3f456b99d4783fd5f2aeb363ce6d7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2c3f456b99d4783fd5f2aeb363ce6d7 columnFamilyName testFamily 2024-11-09T21:42:49,260 DEBUG [StoreOpener-d2c3f456b99d4783fd5f2aeb363ce6d7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:42:49,261 INFO [StoreOpener-d2c3f456b99d4783fd5f2aeb363ce6d7-1 {}] regionserver.HStore(327): Store=d2c3f456b99d4783fd5f2aeb363ce6d7/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:42:49,261 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for d2c3f456b99d4783fd5f2aeb363ce6d7 2024-11-09T21:42:49,262 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testRetryOutOfOrderScannerNextException/d2c3f456b99d4783fd5f2aeb363ce6d7 2024-11-09T21:42:49,263 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testRetryOutOfOrderScannerNextException/d2c3f456b99d4783fd5f2aeb363ce6d7 2024-11-09T21:42:49,264 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for d2c3f456b99d4783fd5f2aeb363ce6d7 2024-11-09T21:42:49,264 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for d2c3f456b99d4783fd5f2aeb363ce6d7 2024-11-09T21:42:49,267 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for d2c3f456b99d4783fd5f2aeb363ce6d7 2024-11-09T21:42:49,273 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testRetryOutOfOrderScannerNextException/d2c3f456b99d4783fd5f2aeb363ce6d7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:42:49,274 INFO [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened d2c3f456b99d4783fd5f2aeb363ce6d7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59688672, jitterRate=-0.11056947708129883}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:42:49,274 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d2c3f456b99d4783fd5f2aeb363ce6d7 2024-11-09T21:42:49,276 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for d2c3f456b99d4783fd5f2aeb363ce6d7: Running coprocessor pre-open hook at 1731188569254Writing region info on filesystem at 1731188569254Initializing all the Stores at 1731188569255 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731188569256 (+1 ms)Cleaning up temporary data from old regions at 1731188569264 (+8 ms)Running coprocessor post-open hooks at 1731188569274 (+10 ms)Region opened successfully at 1731188569275 (+1 ms) 2024-11-09T21:42:49,278 INFO [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1731188568032.d2c3f456b99d4783fd5f2aeb363ce6d7., pid=15, masterSystemTime=1731188569245 2024-11-09T21:42:49,281 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1731188568032.d2c3f456b99d4783fd5f2aeb363ce6d7. 2024-11-09T21:42:49,281 INFO [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1731188568032.d2c3f456b99d4783fd5f2aeb363ce6d7. 2024-11-09T21:42:49,282 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=d2c3f456b99d4783fd5f2aeb363ce6d7, regionState=OPEN, openSeqNum=2, regionLocation=d86ff69d6719,46403,1731188512651 2024-11-09T21:42:49,284 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2c3f456b99d4783fd5f2aeb363ce6d7, server=d86ff69d6719,46403,1731188512651 because future has completed 2024-11-09T21:42:49,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-09T21:42:49,290 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure d2c3f456b99d4783fd5f2aeb363ce6d7, server=d86ff69d6719,46403,1731188512651 in 195 msec 2024-11-09T21:42:49,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-09T21:42:49,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=d2c3f456b99d4783fd5f2aeb363ce6d7, ASSIGN in 363 msec 2024-11-09T21:42:49,294 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T21:42:49,295 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731188569294"}]},"ts":"1731188569294"} 2024-11-09T21:42:49,297 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-11-09T21:42:49,299 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T21:42:49,302 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.2670 sec 2024-11-09T21:42:49,761 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-11-09T21:42:51,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T21:42:51,057 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-11-09T21:42:51,057 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-11-09T21:42:51,058 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T21:42:51,067 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-11-09T21:42:51,067 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T21:42:51,067 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-11-09T21:42:51,071 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1731188568032.d2c3f456b99d4783fd5f2aeb363ce6d7., hostname=d86ff69d6719,46403,1731188512651, seqNum=2] 2024-11-09T21:42:51,073 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T21:42:51,081 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-09T21:42:51,082 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-11-09T21:42:51,085 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-09T21:42:51,088 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-09T21:42:51,089 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-09T21:42:51,090 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46403 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-8853385579539202023 2024-11-09T21:42:51,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46403 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:56192 deadline: 1731188574089, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T21:42:51,094 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-09T21:42:51,096 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-09T21:42:51,097 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-09T21:42:51,098 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-09T21:42:51,099 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46403 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=-8853385579539202021 2024-11-09T21:42:51,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46403 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:56192 deadline: 1731188574098, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T21:42:51,103 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-09T21:42:51,103 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46403 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=-8853385579539202020 2024-11-09T21:42:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46403 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:56192 deadline: 1731188574102, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T21:42:51,107 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-09T21:42:51,108 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46403 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=-8853385579539202019 2024-11-09T21:42:51,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46403 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 44 connection: 172.17.0.2:56192 deadline: 1731188574106, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-09T21:42:51,111 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-09T21:42:51,119 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=223 (was 225), OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=68 (was 68), ProcessCount=11 (was 11), AvailableMemoryMB=17838 (was 17842) 2024-11-09T21:42:51,127 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=68, ProcessCount=11, AvailableMemoryMB=17838 2024-11-09T21:42:51,130 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T21:42:51,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-11-09T21:42:51,134 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T21:42:51,134 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:42:51,134 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-11-09T21:42:51,136 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T21:42:51,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-09T21:42:51,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741843_1019 (size=418) 2024-11-09T21:42:51,552 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ab051e55337f5a7f78d0c70c373a3851, NAME => 'testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118 2024-11-09T21:42:51,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741844_1020 (size=69) 2024-11-09T21:42:51,968 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:42:51,968 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing ab051e55337f5a7f78d0c70c373a3851, disabling compactions & flushes 2024-11-09T21:42:51,968 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851. 2024-11-09T21:42:51,968 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851. 2024-11-09T21:42:51,969 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851. after waiting 0 ms 2024-11-09T21:42:51,969 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851. 2024-11-09T21:42:51,969 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851. 2024-11-09T21:42:51,969 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for ab051e55337f5a7f78d0c70c373a3851: Waiting for close lock at 1731188571968Disabling compacts and flushes for region at 1731188571968Disabling writes for close at 1731188571969 (+1 ms)Writing region close event to WAL at 1731188571969Closed at 1731188571969 2024-11-09T21:42:51,974 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T21:42:51,975 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1731188571975"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731188571975"}]},"ts":"1731188571975"} 2024-11-09T21:42:51,978 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T21:42:51,980 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T21:42:51,980 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731188571980"}]},"ts":"1731188571980"} 2024-11-09T21:42:51,982 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-09T21:42:51,983 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=ab051e55337f5a7f78d0c70c373a3851, ASSIGN}] 2024-11-09T21:42:51,984 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=ab051e55337f5a7f78d0c70c373a3851, ASSIGN 2024-11-09T21:42:51,985 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=ab051e55337f5a7f78d0c70c373a3851, ASSIGN; state=OFFLINE, location=d86ff69d6719,46403,1731188512651; forceNewPlan=false, retain=false 2024-11-09T21:42:52,136 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=ab051e55337f5a7f78d0c70c373a3851, regionState=OPENING, regionLocation=d86ff69d6719,46403,1731188512651 2024-11-09T21:42:52,142 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=ab051e55337f5a7f78d0c70c373a3851, ASSIGN because future has completed 2024-11-09T21:42:52,142 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab051e55337f5a7f78d0c70c373a3851, server=d86ff69d6719,46403,1731188512651}] 2024-11-09T21:42:52,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-09T21:42:52,301 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-09T21:42:52,301 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-11-09T21:42:52,310 INFO [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851. 2024-11-09T21:42:52,310 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => ab051e55337f5a7f78d0c70c373a3851, NAME => 'testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851.', STARTKEY => '', ENDKEY => ''} 2024-11-09T21:42:52,310 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner ab051e55337f5a7f78d0c70c373a3851 2024-11-09T21:42:52,310 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:42:52,310 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for ab051e55337f5a7f78d0c70c373a3851 2024-11-09T21:42:52,310 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for ab051e55337f5a7f78d0c70c373a3851 2024-11-09T21:42:52,312 INFO [StoreOpener-ab051e55337f5a7f78d0c70c373a3851-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region ab051e55337f5a7f78d0c70c373a3851 2024-11-09T21:42:52,314 INFO [StoreOpener-ab051e55337f5a7f78d0c70c373a3851-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab051e55337f5a7f78d0c70c373a3851 columnFamilyName testFamily 2024-11-09T21:42:52,314 DEBUG [StoreOpener-ab051e55337f5a7f78d0c70c373a3851-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:42:52,315 INFO [StoreOpener-ab051e55337f5a7f78d0c70c373a3851-1 {}] regionserver.HStore(327): Store=ab051e55337f5a7f78d0c70c373a3851/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:42:52,315 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for ab051e55337f5a7f78d0c70c373a3851 2024-11-09T21:42:52,316 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnOpenScanner/ab051e55337f5a7f78d0c70c373a3851 2024-11-09T21:42:52,317 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnOpenScanner/ab051e55337f5a7f78d0c70c373a3851 2024-11-09T21:42:52,318 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for ab051e55337f5a7f78d0c70c373a3851 2024-11-09T21:42:52,318 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for ab051e55337f5a7f78d0c70c373a3851 2024-11-09T21:42:52,320 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for ab051e55337f5a7f78d0c70c373a3851 2024-11-09T21:42:52,323 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnOpenScanner/ab051e55337f5a7f78d0c70c373a3851/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:42:52,324 INFO [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened ab051e55337f5a7f78d0c70c373a3851; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67985753, jitterRate=0.013066664338111877}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:42:52,324 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ab051e55337f5a7f78d0c70c373a3851 2024-11-09T21:42:52,325 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for ab051e55337f5a7f78d0c70c373a3851: Running coprocessor pre-open hook at 1731188572310Writing region info on filesystem at 1731188572310Initializing all the Stores at 1731188572312 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731188572312Cleaning up temporary data from old regions at 1731188572318 (+6 ms)Running coprocessor post-open hooks at 1731188572324 (+6 ms)Region opened successfully at 1731188572325 (+1 ms) 2024-11-09T21:42:52,327 INFO [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851., pid=18, masterSystemTime=1731188572298 2024-11-09T21:42:52,329 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851. 2024-11-09T21:42:52,329 INFO [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851. 2024-11-09T21:42:52,330 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=ab051e55337f5a7f78d0c70c373a3851, regionState=OPEN, openSeqNum=2, regionLocation=d86ff69d6719,46403,1731188512651 2024-11-09T21:42:52,333 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab051e55337f5a7f78d0c70c373a3851, server=d86ff69d6719,46403,1731188512651 because future has completed 2024-11-09T21:42:52,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-09T21:42:52,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure ab051e55337f5a7f78d0c70c373a3851, server=d86ff69d6719,46403,1731188512651 in 192 msec 2024-11-09T21:42:52,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-09T21:42:52,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=ab051e55337f5a7f78d0c70c373a3851, ASSIGN in 354 msec 2024-11-09T21:42:52,341 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T21:42:52,342 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731188572341"}]},"ts":"1731188572341"} 2024-11-09T21:42:52,344 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-09T21:42:52,345 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T21:42:52,347 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.2150 sec 2024-11-09T21:42:54,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-09T21:42:54,156 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-11-09T21:42:54,156 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-09T21:42:54,157 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T21:42:54,163 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-09T21:42:54,163 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T21:42:54,163 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-11-09T21:42:54,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851., hostname=d86ff69d6719,46403,1731188512651, seqNum=2] 2024-11-09T21:42:54,179 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-09T21:42:54,180 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-09T21:42:54,184 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-09T21:42:54,186 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46403 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-09T21:42:54,757 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T21:42:55,188 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851., hostname=d86ff69d6719,46403,1731188512651, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851., hostname=d86ff69d6719,46403,1731188512651, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d86ff69d6719:46403 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1002ms, rpcTimeout=1000ms 2024-11-09T21:42:55,188 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851., hostname=d86ff69d6719,46403,1731188512651, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1002ms, rpcTimeout=1000ms 2024-11-09T21:42:55,188 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851., hostname=d86ff69d6719,46403,1731188512651, seqNum=2 from cache 2024-11-09T21:42:55,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 89 bytes 2024-11-09T21:42:55,764 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-11-09T21:42:56,204 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851., hostname=d86ff69d6719,46403,1731188512651, seqNum=2] 2024-11-09T21:42:56,208 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46403 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-09T21:42:57,187 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851., hostname=d86ff69d6719,46403,1731188512651, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851., hostname=d86ff69d6719,46403,1731188512651, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d86ff69d6719:46403 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=980ms, rpcTimeout=979ms 2024-11-09T21:42:57,188 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851., hostname=d86ff69d6719,46403,1731188512651, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=980ms, rpcTimeout=979ms 2024-11-09T21:42:57,188 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851., hostname=d86ff69d6719,46403,1731188512651, seqNum=2 from cache 2024-11-09T21:42:57,204 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 223), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=58 (was 68), ProcessCount=11 (was 11), AvailableMemoryMB=17833 (was 17838) 2024-11-09T21:42:57,214 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=58, ProcessCount=11, AvailableMemoryMB=17832 2024-11-09T21:42:57,216 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T21:42:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-11-09T21:42:57,220 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T21:42:57,220 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:42:57,220 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-11-09T21:42:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T21:42:57,221 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T21:42:57,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741845_1021 (size=416) 2024-11-09T21:42:57,633 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1324ca3e02d44073ea072dea2fead16a, NAME => 'testNormalScanTimeoutOnNextAsync,,1731188577216.1324ca3e02d44073ea072dea2fead16a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118 2024-11-09T21:42:57,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741846_1022 (size=67) 2024-11-09T21:42:57,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 89 bytes 2024-11-09T21:42:58,044 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1731188577216.1324ca3e02d44073ea072dea2fead16a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:42:58,044 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 1324ca3e02d44073ea072dea2fead16a, disabling compactions & flushes 2024-11-09T21:42:58,044 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1731188577216.1324ca3e02d44073ea072dea2fead16a. 2024-11-09T21:42:58,045 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1731188577216.1324ca3e02d44073ea072dea2fead16a. 2024-11-09T21:42:58,045 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1731188577216.1324ca3e02d44073ea072dea2fead16a. after waiting 0 ms 2024-11-09T21:42:58,045 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1731188577216.1324ca3e02d44073ea072dea2fead16a. 2024-11-09T21:42:58,045 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1731188577216.1324ca3e02d44073ea072dea2fead16a. 2024-11-09T21:42:58,046 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1324ca3e02d44073ea072dea2fead16a: Waiting for close lock at 1731188578044Disabling compacts and flushes for region at 1731188578044Disabling writes for close at 1731188578045 (+1 ms)Writing region close event to WAL at 1731188578045Closed at 1731188578045 2024-11-09T21:42:58,049 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T21:42:58,049 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1731188577216.1324ca3e02d44073ea072dea2fead16a.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1731188578049"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731188578049"}]},"ts":"1731188578049"} 2024-11-09T21:42:58,053 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T21:42:58,054 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T21:42:58,054 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731188578054"}]},"ts":"1731188578054"} 2024-11-09T21:42:58,057 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-09T21:42:58,058 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=1324ca3e02d44073ea072dea2fead16a, ASSIGN}] 2024-11-09T21:42:58,060 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=1324ca3e02d44073ea072dea2fead16a, ASSIGN 2024-11-09T21:42:58,061 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=1324ca3e02d44073ea072dea2fead16a, ASSIGN; state=OFFLINE, location=d86ff69d6719,46403,1731188512651; forceNewPlan=false, retain=false 2024-11-09T21:42:58,213 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=1324ca3e02d44073ea072dea2fead16a, regionState=OPENING, regionLocation=d86ff69d6719,46403,1731188512651 2024-11-09T21:42:58,220 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=1324ca3e02d44073ea072dea2fead16a, ASSIGN because future has completed 2024-11-09T21:42:58,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1324ca3e02d44073ea072dea2fead16a, server=d86ff69d6719,46403,1731188512651}] 2024-11-09T21:42:58,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T21:42:58,390 INFO [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1731188577216.1324ca3e02d44073ea072dea2fead16a. 2024-11-09T21:42:58,390 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 1324ca3e02d44073ea072dea2fead16a, NAME => 'testNormalScanTimeoutOnNextAsync,,1731188577216.1324ca3e02d44073ea072dea2fead16a.', STARTKEY => '', ENDKEY => ''} 2024-11-09T21:42:58,391 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync 1324ca3e02d44073ea072dea2fead16a 2024-11-09T21:42:58,391 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1731188577216.1324ca3e02d44073ea072dea2fead16a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:42:58,391 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 1324ca3e02d44073ea072dea2fead16a 2024-11-09T21:42:58,391 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 1324ca3e02d44073ea072dea2fead16a 2024-11-09T21:42:58,393 INFO [StoreOpener-1324ca3e02d44073ea072dea2fead16a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 1324ca3e02d44073ea072dea2fead16a 2024-11-09T21:42:58,395 INFO [StoreOpener-1324ca3e02d44073ea072dea2fead16a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1324ca3e02d44073ea072dea2fead16a columnFamilyName testFamily 2024-11-09T21:42:58,395 DEBUG [StoreOpener-1324ca3e02d44073ea072dea2fead16a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:42:58,396 INFO [StoreOpener-1324ca3e02d44073ea072dea2fead16a-1 {}] regionserver.HStore(327): Store=1324ca3e02d44073ea072dea2fead16a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:42:58,396 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 1324ca3e02d44073ea072dea2fead16a 2024-11-09T21:42:58,397 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnNextAsync/1324ca3e02d44073ea072dea2fead16a 2024-11-09T21:42:58,397 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnNextAsync/1324ca3e02d44073ea072dea2fead16a 2024-11-09T21:42:58,398 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 1324ca3e02d44073ea072dea2fead16a 2024-11-09T21:42:58,398 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 1324ca3e02d44073ea072dea2fead16a 2024-11-09T21:42:58,400 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 1324ca3e02d44073ea072dea2fead16a 2024-11-09T21:42:58,403 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnNextAsync/1324ca3e02d44073ea072dea2fead16a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:42:58,404 INFO [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 1324ca3e02d44073ea072dea2fead16a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72567541, jitterRate=0.08134062588214874}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:42:58,404 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1324ca3e02d44073ea072dea2fead16a 2024-11-09T21:42:58,404 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 1324ca3e02d44073ea072dea2fead16a: Running coprocessor pre-open hook at 1731188578391Writing region info on filesystem at 1731188578391Initializing all the Stores at 1731188578393 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731188578393Cleaning up temporary data from old regions at 1731188578398 (+5 ms)Running coprocessor post-open hooks at 1731188578404 (+6 ms)Region opened successfully at 1731188578404 2024-11-09T21:42:58,405 INFO [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1731188577216.1324ca3e02d44073ea072dea2fead16a., pid=21, masterSystemTime=1731188578376 2024-11-09T21:42:58,408 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1731188577216.1324ca3e02d44073ea072dea2fead16a. 2024-11-09T21:42:58,408 INFO [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1731188577216.1324ca3e02d44073ea072dea2fead16a. 2024-11-09T21:42:58,409 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=1324ca3e02d44073ea072dea2fead16a, regionState=OPEN, openSeqNum=2, regionLocation=d86ff69d6719,46403,1731188512651 2024-11-09T21:42:58,412 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1324ca3e02d44073ea072dea2fead16a, server=d86ff69d6719,46403,1731188512651 because future has completed 2024-11-09T21:42:58,416 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-09T21:42:58,416 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 1324ca3e02d44073ea072dea2fead16a, server=d86ff69d6719,46403,1731188512651 in 192 msec 2024-11-09T21:42:58,419 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-09T21:42:58,419 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=1324ca3e02d44073ea072dea2fead16a, ASSIGN in 358 msec 2024-11-09T21:42:58,420 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T21:42:58,421 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731188578420"}]},"ts":"1731188578420"} 2024-11-09T21:42:58,423 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-09T21:42:58,424 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T21:42:58,427 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 1.2080 sec 2024-11-09T21:43:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T21:43:00,249 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-11-09T21:43:00,249 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-09T21:43:00,250 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T21:43:00,261 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-09T21:43:00,262 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T21:43:00,262 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-11-09T21:43:00,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1731188577216.1324ca3e02d44073ea072dea2fead16a., hostname=d86ff69d6719,46403,1731188512651, seqNum=2] 2024-11-09T21:43:00,279 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-09T21:43:00,279 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-09T21:43:00,282 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T21:43:00,285 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-09T21:43:00,286 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-09T21:43:00,286 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46403 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-09T21:43:01,766 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-11-09T21:43:02,301 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-09T21:43:02,301 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-09T21:43:02,302 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-09T21:43:02,302 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-09T21:43:03,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46403 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8853385579539202008' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-09T21:43:03,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46403 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:56192 deadline: 1731188584289, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8853385579539202008' does not exist 2024-11-09T21:43:03,297 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to d86ff69d6719,46403,1731188512651 for closing scanner id = -8853385579539202008 for 1324ca3e02d44073ea072dea2fead16a of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8853385579539202008' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:43:03,302 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=53 (was 58), ProcessCount=11 (was 11), AvailableMemoryMB=17828 (was 17832) 2024-11-09T21:43:03,310 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=53, ProcessCount=11, AvailableMemoryMB=17828 2024-11-09T21:43:03,312 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T21:43:03,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-11-09T21:43:03,316 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T21:43:03,316 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:43:03,316 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-11-09T21:43:03,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-09T21:43:03,318 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T21:43:03,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741847_1023 (size=411) 2024-11-09T21:43:03,735 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => be5d9ef20c6a1539c9edfca19c08ac41, NAME => 'testNormalScanTimeoutOnNext,,1731188583312.be5d9ef20c6a1539c9edfca19c08ac41.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118 2024-11-09T21:43:03,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741848_1024 (size=62) 2024-11-09T21:43:03,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 83 bytes 2024-11-09T21:43:03,896 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T21:43:04,152 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1731188583312.be5d9ef20c6a1539c9edfca19c08ac41.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:43:04,153 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing be5d9ef20c6a1539c9edfca19c08ac41, disabling compactions & flushes 2024-11-09T21:43:04,153 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1731188583312.be5d9ef20c6a1539c9edfca19c08ac41. 2024-11-09T21:43:04,153 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1731188583312.be5d9ef20c6a1539c9edfca19c08ac41. 2024-11-09T21:43:04,153 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1731188583312.be5d9ef20c6a1539c9edfca19c08ac41. after waiting 0 ms 2024-11-09T21:43:04,153 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1731188583312.be5d9ef20c6a1539c9edfca19c08ac41. 2024-11-09T21:43:04,154 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1731188583312.be5d9ef20c6a1539c9edfca19c08ac41. 2024-11-09T21:43:04,154 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for be5d9ef20c6a1539c9edfca19c08ac41: Waiting for close lock at 1731188584153Disabling compacts and flushes for region at 1731188584153Disabling writes for close at 1731188584153Writing region close event to WAL at 1731188584154 (+1 ms)Closed at 1731188584154 2024-11-09T21:43:04,157 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T21:43:04,158 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1731188583312.be5d9ef20c6a1539c9edfca19c08ac41.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1731188584157"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731188584157"}]},"ts":"1731188584157"} 2024-11-09T21:43:04,162 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T21:43:04,165 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T21:43:04,165 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731188584165"}]},"ts":"1731188584165"} 2024-11-09T21:43:04,169 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-09T21:43:04,169 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=be5d9ef20c6a1539c9edfca19c08ac41, ASSIGN}] 2024-11-09T21:43:04,171 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=be5d9ef20c6a1539c9edfca19c08ac41, ASSIGN 2024-11-09T21:43:04,172 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=be5d9ef20c6a1539c9edfca19c08ac41, ASSIGN; state=OFFLINE, location=d86ff69d6719,46403,1731188512651; forceNewPlan=false, retain=false 2024-11-09T21:43:04,324 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=be5d9ef20c6a1539c9edfca19c08ac41, regionState=OPENING, regionLocation=d86ff69d6719,46403,1731188512651 2024-11-09T21:43:04,331 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=be5d9ef20c6a1539c9edfca19c08ac41, ASSIGN because future has completed 2024-11-09T21:43:04,332 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure be5d9ef20c6a1539c9edfca19c08ac41, server=d86ff69d6719,46403,1731188512651}] 2024-11-09T21:43:04,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-09T21:43:04,500 INFO [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1731188583312.be5d9ef20c6a1539c9edfca19c08ac41. 2024-11-09T21:43:04,501 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => be5d9ef20c6a1539c9edfca19c08ac41, NAME => 'testNormalScanTimeoutOnNext,,1731188583312.be5d9ef20c6a1539c9edfca19c08ac41.', STARTKEY => '', ENDKEY => ''} 2024-11-09T21:43:04,501 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext be5d9ef20c6a1539c9edfca19c08ac41 2024-11-09T21:43:04,501 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1731188583312.be5d9ef20c6a1539c9edfca19c08ac41.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:43:04,501 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for be5d9ef20c6a1539c9edfca19c08ac41 2024-11-09T21:43:04,501 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for be5d9ef20c6a1539c9edfca19c08ac41 2024-11-09T21:43:04,503 INFO [StoreOpener-be5d9ef20c6a1539c9edfca19c08ac41-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region be5d9ef20c6a1539c9edfca19c08ac41 2024-11-09T21:43:04,505 INFO [StoreOpener-be5d9ef20c6a1539c9edfca19c08ac41-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be5d9ef20c6a1539c9edfca19c08ac41 columnFamilyName testFamily 2024-11-09T21:43:04,505 DEBUG [StoreOpener-be5d9ef20c6a1539c9edfca19c08ac41-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:43:04,506 INFO [StoreOpener-be5d9ef20c6a1539c9edfca19c08ac41-1 {}] regionserver.HStore(327): Store=be5d9ef20c6a1539c9edfca19c08ac41/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:43:04,506 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for be5d9ef20c6a1539c9edfca19c08ac41 2024-11-09T21:43:04,508 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnNext/be5d9ef20c6a1539c9edfca19c08ac41 2024-11-09T21:43:04,508 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnNext/be5d9ef20c6a1539c9edfca19c08ac41 2024-11-09T21:43:04,509 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for be5d9ef20c6a1539c9edfca19c08ac41 2024-11-09T21:43:04,509 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for be5d9ef20c6a1539c9edfca19c08ac41 2024-11-09T21:43:04,512 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for be5d9ef20c6a1539c9edfca19c08ac41 2024-11-09T21:43:04,515 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnNext/be5d9ef20c6a1539c9edfca19c08ac41/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:43:04,517 INFO [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened be5d9ef20c6a1539c9edfca19c08ac41; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66471082, jitterRate=-0.009503692388534546}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:43:04,517 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for be5d9ef20c6a1539c9edfca19c08ac41 2024-11-09T21:43:04,518 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for be5d9ef20c6a1539c9edfca19c08ac41: Running coprocessor pre-open hook at 1731188584501Writing region info on filesystem at 1731188584501Initializing all the Stores at 1731188584503 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731188584503Cleaning up temporary data from old regions at 1731188584509 (+6 ms)Running coprocessor post-open hooks at 1731188584517 (+8 ms)Region opened successfully at 1731188584518 (+1 ms) 2024-11-09T21:43:04,519 INFO [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1731188583312.be5d9ef20c6a1539c9edfca19c08ac41., pid=24, masterSystemTime=1731188584488 2024-11-09T21:43:04,522 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1731188583312.be5d9ef20c6a1539c9edfca19c08ac41. 2024-11-09T21:43:04,523 INFO [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1731188583312.be5d9ef20c6a1539c9edfca19c08ac41. 2024-11-09T21:43:04,523 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=be5d9ef20c6a1539c9edfca19c08ac41, regionState=OPEN, openSeqNum=2, regionLocation=d86ff69d6719,46403,1731188512651 2024-11-09T21:43:04,526 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure be5d9ef20c6a1539c9edfca19c08ac41, server=d86ff69d6719,46403,1731188512651 because future has completed 2024-11-09T21:43:04,530 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-09T21:43:04,530 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure be5d9ef20c6a1539c9edfca19c08ac41, server=d86ff69d6719,46403,1731188512651 in 195 msec 2024-11-09T21:43:04,533 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-09T21:43:04,533 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=be5d9ef20c6a1539c9edfca19c08ac41, ASSIGN in 361 msec 2024-11-09T21:43:04,534 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T21:43:04,534 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731188584534"}]},"ts":"1731188584534"} 2024-11-09T21:43:04,537 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-09T21:43:04,538 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T21:43:04,540 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.2260 sec 2024-11-09T21:43:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-09T21:43:06,359 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-11-09T21:43:06,359 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-09T21:43:06,360 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T21:43:06,368 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-09T21:43:06,369 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T21:43:06,369 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-11-09T21:43:06,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1731188583312.be5d9ef20c6a1539c9edfca19c08ac41., hostname=d86ff69d6719,46403,1731188512651, seqNum=2] 2024-11-09T21:43:06,383 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-09T21:43:06,383 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-09T21:43:06,386 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T21:43:06,389 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-09T21:43:06,390 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-09T21:43:06,391 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46403 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-09T21:43:06,448 INFO [master/d86ff69d6719:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-09T21:43:06,448 INFO [master/d86ff69d6719:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-09T21:43:07,773 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-11-09T21:43:09,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46403 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8853385579539202004' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-09T21:43:09,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46403 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:56192 deadline: 1731188590398, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8853385579539202004' does not exist 2024-11-09T21:43:09,400 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to d86ff69d6719,46403,1731188512651 for closing scanner id = -8853385579539202004 for be5d9ef20c6a1539c9edfca19c08ac41 of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8853385579539202004' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:43:09,404 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=49 (was 53), ProcessCount=11 (was 11), AvailableMemoryMB=17823 (was 17828) 2024-11-09T21:43:09,411 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=49, ProcessCount=11, AvailableMemoryMB=17823 2024-11-09T21:43:09,413 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43251 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T21:43:09,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43251 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-11-09T21:43:09,416 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T21:43:09,416 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:43:09,416 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43251 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-11-09T21:43:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T21:43:09,417 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T21:43:09,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741849_1025 (size=419) 2024-11-09T21:43:09,833 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 82505c28df963a1a69ff02aae6cb9d53, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731188589413.82505c28df963a1a69ff02aae6cb9d53.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118 2024-11-09T21:43:09,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741850_1026 (size=70) 2024-11-09T21:43:09,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 83 bytes 2024-11-09T21:43:10,005 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T21:43:10,245 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731188589413.82505c28df963a1a69ff02aae6cb9d53.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:43:10,246 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 82505c28df963a1a69ff02aae6cb9d53, disabling compactions & flushes 2024-11-09T21:43:10,246 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731188589413.82505c28df963a1a69ff02aae6cb9d53. 2024-11-09T21:43:10,246 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731188589413.82505c28df963a1a69ff02aae6cb9d53. 2024-11-09T21:43:10,246 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731188589413.82505c28df963a1a69ff02aae6cb9d53. after waiting 0 ms 2024-11-09T21:43:10,246 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731188589413.82505c28df963a1a69ff02aae6cb9d53. 2024-11-09T21:43:10,246 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731188589413.82505c28df963a1a69ff02aae6cb9d53. 2024-11-09T21:43:10,246 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 82505c28df963a1a69ff02aae6cb9d53: Waiting for close lock at 1731188590245Disabling compacts and flushes for region at 1731188590245Disabling writes for close at 1731188590246 (+1 ms)Writing region close event to WAL at 1731188590246Closed at 1731188590246 2024-11-09T21:43:10,309 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T21:43:10,310 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731188589413.82505c28df963a1a69ff02aae6cb9d53.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1731188590309"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731188590309"}]},"ts":"1731188590309"} 2024-11-09T21:43:10,315 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T21:43:10,317 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T21:43:10,318 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731188590317"}]},"ts":"1731188590317"} 2024-11-09T21:43:10,321 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-09T21:43:10,321 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=82505c28df963a1a69ff02aae6cb9d53, ASSIGN}] 2024-11-09T21:43:10,323 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=82505c28df963a1a69ff02aae6cb9d53, ASSIGN 2024-11-09T21:43:10,324 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=82505c28df963a1a69ff02aae6cb9d53, ASSIGN; state=OFFLINE, location=d86ff69d6719,46403,1731188512651; forceNewPlan=false, retain=false 2024-11-09T21:43:10,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T21:43:10,475 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=82505c28df963a1a69ff02aae6cb9d53, regionState=OPENING, regionLocation=d86ff69d6719,46403,1731188512651 2024-11-09T21:43:10,483 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=82505c28df963a1a69ff02aae6cb9d53, ASSIGN because future has completed 2024-11-09T21:43:10,483 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82505c28df963a1a69ff02aae6cb9d53, server=d86ff69d6719,46403,1731188512651}] 2024-11-09T21:43:10,648 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731188589413.82505c28df963a1a69ff02aae6cb9d53. 2024-11-09T21:43:10,649 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 82505c28df963a1a69ff02aae6cb9d53, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731188589413.82505c28df963a1a69ff02aae6cb9d53.', STARTKEY => '', ENDKEY => ''} 2024-11-09T21:43:10,649 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync 82505c28df963a1a69ff02aae6cb9d53 2024-11-09T21:43:10,650 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731188589413.82505c28df963a1a69ff02aae6cb9d53.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:43:10,650 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 82505c28df963a1a69ff02aae6cb9d53 2024-11-09T21:43:10,650 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 82505c28df963a1a69ff02aae6cb9d53 2024-11-09T21:43:10,653 INFO [StoreOpener-82505c28df963a1a69ff02aae6cb9d53-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 82505c28df963a1a69ff02aae6cb9d53 2024-11-09T21:43:10,655 INFO [StoreOpener-82505c28df963a1a69ff02aae6cb9d53-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82505c28df963a1a69ff02aae6cb9d53 columnFamilyName testFamily 2024-11-09T21:43:10,656 DEBUG [StoreOpener-82505c28df963a1a69ff02aae6cb9d53-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:43:10,657 INFO [StoreOpener-82505c28df963a1a69ff02aae6cb9d53-1 {}] regionserver.HStore(327): Store=82505c28df963a1a69ff02aae6cb9d53/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:43:10,657 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 82505c28df963a1a69ff02aae6cb9d53 2024-11-09T21:43:10,658 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/82505c28df963a1a69ff02aae6cb9d53 2024-11-09T21:43:10,659 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/82505c28df963a1a69ff02aae6cb9d53 2024-11-09T21:43:10,660 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 82505c28df963a1a69ff02aae6cb9d53 2024-11-09T21:43:10,660 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 82505c28df963a1a69ff02aae6cb9d53 2024-11-09T21:43:10,663 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 82505c28df963a1a69ff02aae6cb9d53 2024-11-09T21:43:10,666 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/82505c28df963a1a69ff02aae6cb9d53/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:43:10,667 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened 82505c28df963a1a69ff02aae6cb9d53; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60039179, jitterRate=-0.10534651577472687}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:43:10,667 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 82505c28df963a1a69ff02aae6cb9d53 2024-11-09T21:43:10,667 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 82505c28df963a1a69ff02aae6cb9d53: Running coprocessor pre-open hook at 1731188590650Writing region info on filesystem at 1731188590650Initializing all the Stores at 1731188590652 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731188590652Cleaning up temporary data from old regions at 1731188590660 (+8 ms)Running coprocessor post-open hooks at 1731188590667 (+7 ms)Region opened successfully at 1731188590667 2024-11-09T21:43:10,668 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731188589413.82505c28df963a1a69ff02aae6cb9d53., pid=27, masterSystemTime=1731188590637 2024-11-09T21:43:10,671 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731188589413.82505c28df963a1a69ff02aae6cb9d53. 2024-11-09T21:43:10,671 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731188589413.82505c28df963a1a69ff02aae6cb9d53. 2024-11-09T21:43:10,671 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=82505c28df963a1a69ff02aae6cb9d53, regionState=OPEN, openSeqNum=2, regionLocation=d86ff69d6719,46403,1731188512651 2024-11-09T21:43:10,674 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82505c28df963a1a69ff02aae6cb9d53, server=d86ff69d6719,46403,1731188512651 because future has completed 2024-11-09T21:43:10,677 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-09T21:43:10,677 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 82505c28df963a1a69ff02aae6cb9d53, server=d86ff69d6719,46403,1731188512651 in 192 msec 2024-11-09T21:43:10,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-09T21:43:10,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=82505c28df963a1a69ff02aae6cb9d53, ASSIGN in 356 msec 2024-11-09T21:43:10,681 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T21:43:10,681 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731188590681"}]},"ts":"1731188590681"} 2024-11-09T21:43:10,684 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-09T21:43:10,685 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T21:43:10,687 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.2720 sec 2024-11-09T21:43:12,301 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-09T21:43:12,301 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-09T21:43:12,302 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-09T21:43:12,302 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-09T21:43:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T21:43:12,458 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-11-09T21:43:12,459 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-09T21:43:12,459 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T21:43:12,470 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-09T21:43:12,470 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T21:43:12,471 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-11-09T21:43:12,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731188589413.82505c28df963a1a69ff02aae6cb9d53., hostname=d86ff69d6719,46403,1731188512651, seqNum=2] 2024-11-09T21:43:12,484 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-09T21:43:12,484 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-09T21:43:12,487 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-09T21:43:12,488 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46403 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-09T21:43:13,776 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-11-09T21:43:16,155 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T21:43:18,497 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731188589413.82505c28df963a1a69ff02aae6cb9d53., hostname=d86ff69d6719,46403,1731188512651, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731188589413.82505c28df963a1a69ff02aae6cb9d53., hostname=d86ff69d6719,46403,1731188512651, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d86ff69d6719:46403 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6008ms, rpcTimeout=5999ms 2024-11-09T21:43:18,497 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731188589413.82505c28df963a1a69ff02aae6cb9d53., hostname=d86ff69d6719,46403,1731188512651, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6008ms, rpcTimeout=5999ms 2024-11-09T21:43:18,498 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731188589413.82505c28df963a1a69ff02aae6cb9d53., hostname=d86ff69d6719,46403,1731188512651, seqNum=2 from cache 2024-11-09T21:43:18,513 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=41 (was 49), ProcessCount=11 (was 11), AvailableMemoryMB=17817 (was 17823) 2024-11-09T21:43:18,522 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=41, ProcessCount=11, AvailableMemoryMB=17816 2024-11-09T21:43:18,524 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43251 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T21:43:18,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43251 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-11-09T21:43:18,528 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T21:43:18,528 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:43:18,528 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43251 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-11-09T21:43:18,530 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T21:43:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-09T21:43:18,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741851_1027 (size=407) 2024-11-09T21:43:18,948 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4d94b8668efb90b9ce664fd703f2c8d3, NAME => 'hbase:testMetaScanTimeoutOnNext,,1731188598524.4d94b8668efb90b9ce664fd703f2c8d3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118 2024-11-09T21:43:18,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741852_1028 (size=58) 2024-11-09T21:43:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 89 bytes 2024-11-09T21:43:19,362 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1731188598524.4d94b8668efb90b9ce664fd703f2c8d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:43:19,363 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 4d94b8668efb90b9ce664fd703f2c8d3, disabling compactions & flushes 2024-11-09T21:43:19,363 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1731188598524.4d94b8668efb90b9ce664fd703f2c8d3. 2024-11-09T21:43:19,363 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1731188598524.4d94b8668efb90b9ce664fd703f2c8d3. 2024-11-09T21:43:19,363 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1731188598524.4d94b8668efb90b9ce664fd703f2c8d3. after waiting 0 ms 2024-11-09T21:43:19,363 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1731188598524.4d94b8668efb90b9ce664fd703f2c8d3. 2024-11-09T21:43:19,364 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1731188598524.4d94b8668efb90b9ce664fd703f2c8d3. 2024-11-09T21:43:19,364 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4d94b8668efb90b9ce664fd703f2c8d3: Waiting for close lock at 1731188599363Disabling compacts and flushes for region at 1731188599363Disabling writes for close at 1731188599363Writing region close event to WAL at 1731188599364 (+1 ms)Closed at 1731188599364 2024-11-09T21:43:19,366 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T21:43:19,367 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1731188598524.4d94b8668efb90b9ce664fd703f2c8d3.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1731188599366"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731188599366"}]},"ts":"1731188599366"} 2024-11-09T21:43:19,371 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T21:43:19,373 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T21:43:19,373 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731188599373"}]},"ts":"1731188599373"} 2024-11-09T21:43:19,376 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-09T21:43:19,377 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=4d94b8668efb90b9ce664fd703f2c8d3, ASSIGN}] 2024-11-09T21:43:19,378 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=4d94b8668efb90b9ce664fd703f2c8d3, ASSIGN 2024-11-09T21:43:19,379 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=4d94b8668efb90b9ce664fd703f2c8d3, ASSIGN; state=OFFLINE, location=d86ff69d6719,46403,1731188512651; forceNewPlan=false, retain=false 2024-11-09T21:43:19,531 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=4d94b8668efb90b9ce664fd703f2c8d3, regionState=OPENING, regionLocation=d86ff69d6719,46403,1731188512651 2024-11-09T21:43:19,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-09T21:43:19,537 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=4d94b8668efb90b9ce664fd703f2c8d3, ASSIGN because future has completed 2024-11-09T21:43:19,537 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d94b8668efb90b9ce664fd703f2c8d3, server=d86ff69d6719,46403,1731188512651}] 2024-11-09T21:43:19,703 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1731188598524.4d94b8668efb90b9ce664fd703f2c8d3. 2024-11-09T21:43:19,703 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 4d94b8668efb90b9ce664fd703f2c8d3, NAME => 'hbase:testMetaScanTimeoutOnNext,,1731188598524.4d94b8668efb90b9ce664fd703f2c8d3.', STARTKEY => '', ENDKEY => ''} 2024-11-09T21:43:19,704 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 4d94b8668efb90b9ce664fd703f2c8d3 2024-11-09T21:43:19,704 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1731188598524.4d94b8668efb90b9ce664fd703f2c8d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:43:19,704 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 4d94b8668efb90b9ce664fd703f2c8d3 2024-11-09T21:43:19,704 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 4d94b8668efb90b9ce664fd703f2c8d3 2024-11-09T21:43:19,706 INFO [StoreOpener-4d94b8668efb90b9ce664fd703f2c8d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 4d94b8668efb90b9ce664fd703f2c8d3 2024-11-09T21:43:19,708 INFO [StoreOpener-4d94b8668efb90b9ce664fd703f2c8d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d94b8668efb90b9ce664fd703f2c8d3 columnFamilyName testFamily 2024-11-09T21:43:19,708 DEBUG [StoreOpener-4d94b8668efb90b9ce664fd703f2c8d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:43:19,709 INFO [StoreOpener-4d94b8668efb90b9ce664fd703f2c8d3-1 {}] regionserver.HStore(327): Store=4d94b8668efb90b9ce664fd703f2c8d3/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:43:19,709 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 4d94b8668efb90b9ce664fd703f2c8d3 2024-11-09T21:43:19,710 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnNext/4d94b8668efb90b9ce664fd703f2c8d3 2024-11-09T21:43:19,710 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnNext/4d94b8668efb90b9ce664fd703f2c8d3 2024-11-09T21:43:19,711 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 4d94b8668efb90b9ce664fd703f2c8d3 2024-11-09T21:43:19,711 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 4d94b8668efb90b9ce664fd703f2c8d3 2024-11-09T21:43:19,713 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 4d94b8668efb90b9ce664fd703f2c8d3 2024-11-09T21:43:19,717 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnNext/4d94b8668efb90b9ce664fd703f2c8d3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:43:19,718 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 4d94b8668efb90b9ce664fd703f2c8d3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70864561, jitterRate=0.05596424639225006}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:43:19,718 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4d94b8668efb90b9ce664fd703f2c8d3 2024-11-09T21:43:19,719 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 4d94b8668efb90b9ce664fd703f2c8d3: Running coprocessor pre-open hook at 1731188599704Writing region info on filesystem at 1731188599704Initializing all the Stores at 1731188599705 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731188599705Cleaning up temporary data from old regions at 1731188599711 (+6 ms)Running coprocessor post-open hooks at 1731188599718 (+7 ms)Region opened successfully at 1731188599718 2024-11-09T21:43:19,720 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1731188598524.4d94b8668efb90b9ce664fd703f2c8d3., pid=30, masterSystemTime=1731188599691 2024-11-09T21:43:19,722 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1731188598524.4d94b8668efb90b9ce664fd703f2c8d3. 2024-11-09T21:43:19,723 INFO [RS_OPEN_PRIORITY_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1731188598524.4d94b8668efb90b9ce664fd703f2c8d3. 2024-11-09T21:43:19,723 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=4d94b8668efb90b9ce664fd703f2c8d3, regionState=OPEN, openSeqNum=2, regionLocation=d86ff69d6719,46403,1731188512651 2024-11-09T21:43:19,726 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d94b8668efb90b9ce664fd703f2c8d3, server=d86ff69d6719,46403,1731188512651 because future has completed 2024-11-09T21:43:19,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-09T21:43:19,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 4d94b8668efb90b9ce664fd703f2c8d3, server=d86ff69d6719,46403,1731188512651 in 191 msec 2024-11-09T21:43:19,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-09T21:43:19,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=4d94b8668efb90b9ce664fd703f2c8d3, ASSIGN in 355 msec 2024-11-09T21:43:19,736 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T21:43:19,736 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731188599736"}]},"ts":"1731188599736"} 2024-11-09T21:43:19,738 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-09T21:43:19,739 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T21:43:19,741 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.2140 sec 2024-11-09T21:43:19,779 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-11-09T21:43:21,074 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T21:43:21,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-09T21:43:21,558 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-11-09T21:43:21,558 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-09T21:43:21,560 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T21:43:21,569 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-09T21:43:21,570 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T21:43:21,570 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-11-09T21:43:21,573 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1731188598524.4d94b8668efb90b9ce664fd703f2c8d3., hostname=d86ff69d6719,46403,1731188512651, seqNum=2] 2024-11-09T21:43:21,628 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-09T21:43:21,629 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-09T21:43:21,632 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-09T21:43:21,634 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-09T21:43:21,635 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-09T21:43:21,636 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46403 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-09T21:43:22,301 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-09T21:43:22,301 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-09T21:43:25,208 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T21:43:26,456 DEBUG [master/d86ff69d6719:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-09T21:43:27,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46403 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8853385579539201996' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-09T21:43:27,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46403 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 41 connection: 172.17.0.2:56192 deadline: 1731188613639, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8853385579539201996' does not exist 2024-11-09T21:43:27,640 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to d86ff69d6719,46403,1731188512651 for closing scanner id = -8853385579539201996 for 4d94b8668efb90b9ce664fd703f2c8d3 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '-8853385579539201996' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:43:27,645 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=223 (was 222) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=35 (was 41), ProcessCount=11 (was 11), AvailableMemoryMB=17812 (was 17816) 2024-11-09T21:43:27,654 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=35, ProcessCount=11, AvailableMemoryMB=17811 2024-11-09T21:43:27,656 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T21:43:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-11-09T21:43:27,659 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T21:43:27,659 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:43:27,659 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-11-09T21:43:27,661 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T21:43:27,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-09T21:43:27,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741853_1029 (size=423) 2024-11-09T21:43:28,076 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 16065d24acc6760bfc8ee9d803a26d64, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118 2024-11-09T21:43:28,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741854_1030 (size=74) 2024-11-09T21:43:28,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 83 bytes 2024-11-09T21:43:28,492 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:43:28,492 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 16065d24acc6760bfc8ee9d803a26d64, disabling compactions & flushes 2024-11-09T21:43:28,492 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64. 2024-11-09T21:43:28,492 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64. 2024-11-09T21:43:28,493 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64. after waiting 0 ms 2024-11-09T21:43:28,493 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64. 2024-11-09T21:43:28,493 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64. 2024-11-09T21:43:28,493 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 16065d24acc6760bfc8ee9d803a26d64: Waiting for close lock at 1731188608492Disabling compacts and flushes for region at 1731188608492Disabling writes for close at 1731188608493 (+1 ms)Writing region close event to WAL at 1731188608493Closed at 1731188608493 2024-11-09T21:43:28,497 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T21:43:28,497 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1731188608497"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731188608497"}]},"ts":"1731188608497"} 2024-11-09T21:43:28,502 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T21:43:28,503 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T21:43:28,504 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731188608503"}]},"ts":"1731188608503"} 2024-11-09T21:43:28,506 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-09T21:43:28,506 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=16065d24acc6760bfc8ee9d803a26d64, ASSIGN}] 2024-11-09T21:43:28,508 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=16065d24acc6760bfc8ee9d803a26d64, ASSIGN 2024-11-09T21:43:28,509 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=16065d24acc6760bfc8ee9d803a26d64, ASSIGN; state=OFFLINE, location=d86ff69d6719,46403,1731188512651; forceNewPlan=false, retain=false 2024-11-09T21:43:28,660 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=16065d24acc6760bfc8ee9d803a26d64, regionState=OPENING, regionLocation=d86ff69d6719,46403,1731188512651 2024-11-09T21:43:28,666 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=16065d24acc6760bfc8ee9d803a26d64, ASSIGN because future has completed 2024-11-09T21:43:28,667 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 16065d24acc6760bfc8ee9d803a26d64, server=d86ff69d6719,46403,1731188512651}] 2024-11-09T21:43:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-09T21:43:28,825 INFO [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64. 2024-11-09T21:43:28,825 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 16065d24acc6760bfc8ee9d803a26d64, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64.', STARTKEY => '', ENDKEY => ''} 2024-11-09T21:43:28,825 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 16065d24acc6760bfc8ee9d803a26d64 2024-11-09T21:43:28,825 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:43:28,826 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 16065d24acc6760bfc8ee9d803a26d64 2024-11-09T21:43:28,826 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 16065d24acc6760bfc8ee9d803a26d64 2024-11-09T21:43:28,828 INFO [StoreOpener-16065d24acc6760bfc8ee9d803a26d64-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 16065d24acc6760bfc8ee9d803a26d64 2024-11-09T21:43:28,829 INFO [StoreOpener-16065d24acc6760bfc8ee9d803a26d64-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 16065d24acc6760bfc8ee9d803a26d64 columnFamilyName testFamily 2024-11-09T21:43:28,830 DEBUG [StoreOpener-16065d24acc6760bfc8ee9d803a26d64-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:43:28,830 INFO [StoreOpener-16065d24acc6760bfc8ee9d803a26d64-1 {}] regionserver.HStore(327): Store=16065d24acc6760bfc8ee9d803a26d64/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:43:28,831 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 16065d24acc6760bfc8ee9d803a26d64 2024-11-09T21:43:28,832 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnOpenScannerAsync/16065d24acc6760bfc8ee9d803a26d64 2024-11-09T21:43:28,832 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnOpenScannerAsync/16065d24acc6760bfc8ee9d803a26d64 2024-11-09T21:43:28,833 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 16065d24acc6760bfc8ee9d803a26d64 2024-11-09T21:43:28,833 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 16065d24acc6760bfc8ee9d803a26d64 2024-11-09T21:43:28,835 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 16065d24acc6760bfc8ee9d803a26d64 2024-11-09T21:43:28,837 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnOpenScannerAsync/16065d24acc6760bfc8ee9d803a26d64/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:43:28,838 INFO [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 16065d24acc6760bfc8ee9d803a26d64; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61084726, jitterRate=-0.08976665139198303}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T21:43:28,838 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 16065d24acc6760bfc8ee9d803a26d64 2024-11-09T21:43:28,839 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 16065d24acc6760bfc8ee9d803a26d64: Running coprocessor pre-open hook at 1731188608826Writing region info on filesystem at 1731188608826Initializing all the Stores at 1731188608827 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731188608827Cleaning up temporary data from old regions at 1731188608833 (+6 ms)Running coprocessor post-open hooks at 1731188608838 (+5 ms)Region opened successfully at 1731188608839 (+1 ms) 2024-11-09T21:43:28,840 INFO [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64., pid=33, masterSystemTime=1731188608820 2024-11-09T21:43:28,843 DEBUG [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64. 2024-11-09T21:43:28,843 INFO [RS_OPEN_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64. 2024-11-09T21:43:28,843 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=16065d24acc6760bfc8ee9d803a26d64, regionState=OPEN, openSeqNum=2, regionLocation=d86ff69d6719,46403,1731188512651 2024-11-09T21:43:28,846 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 16065d24acc6760bfc8ee9d803a26d64, server=d86ff69d6719,46403,1731188512651 because future has completed 2024-11-09T21:43:28,851 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-09T21:43:28,852 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 16065d24acc6760bfc8ee9d803a26d64, server=d86ff69d6719,46403,1731188512651 in 181 msec 2024-11-09T21:43:28,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-09T21:43:28,854 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=16065d24acc6760bfc8ee9d803a26d64, ASSIGN in 345 msec 2024-11-09T21:43:28,855 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T21:43:28,855 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731188608855"}]},"ts":"1731188608855"} 2024-11-09T21:43:28,857 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-09T21:43:28,859 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T21:43:28,860 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.2030 sec 2024-11-09T21:43:30,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-09T21:43:30,699 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-11-09T21:43:30,699 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-09T21:43:30,700 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T21:43:30,710 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-09T21:43:30,710 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T21:43:30,710 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-11-09T21:43:30,715 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64., hostname=d86ff69d6719,46403,1731188512651, seqNum=2] 2024-11-09T21:43:30,726 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-09T21:43:30,726 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-09T21:43:30,729 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-09T21:43:30,731 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46403 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-09T21:43:31,738 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64., hostname=d86ff69d6719,46403,1731188512651, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64., hostname=d86ff69d6719,46403,1731188512651, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d86ff69d6719:46403 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms 2024-11-09T21:43:31,738 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64., hostname=d86ff69d6719,46403,1731188512651, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1007ms, rpcTimeout=1000ms 2024-11-09T21:43:31,738 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64., hostname=d86ff69d6719,46403,1731188512651, seqNum=2 from cache 2024-11-09T21:43:31,781 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-11-09T21:43:32,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 89 bytes 2024-11-09T21:43:32,301 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-11-09T21:43:32,301 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-09T21:43:32,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64., hostname=d86ff69d6719,46403,1731188512651, seqNum=2] 2024-11-09T21:43:32,759 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46403 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-09T21:43:33,737 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64., hostname=d86ff69d6719,46403,1731188512651, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64., hostname=d86ff69d6719,46403,1731188512651, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=d86ff69d6719:46403 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=982ms, rpcTimeout=974ms 2024-11-09T21:43:33,738 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64., hostname=d86ff69d6719,46403,1731188512651, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=982ms, rpcTimeout=974ms 2024-11-09T21:43:33,738 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64., hostname=d86ff69d6719,46403,1731188512651, seqNum=2 from cache 2024-11-09T21:43:33,752 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=223 (was 223), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=32 (was 35), ProcessCount=11 (was 11), AvailableMemoryMB=17805 (was 17811) 2024-11-09T21:43:33,752 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T21:43:33,753 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T21:43:33,758 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:43:33,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:43:33,759 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:43:33,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-09T21:43:33,759 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T21:43:33,759 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T21:43:33,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:43:33,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:43:33,760 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:43:33,760 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-09T21:43:33,760 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=836423166, stopped=false 2024-11-09T21:43:33,760 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d86ff69d6719,43251,1731188511956 2024-11-09T21:43:33,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43251-0x10122d5e4550000, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T21:43:33,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46403-0x10122d5e4550001, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T21:43:33,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43251-0x10122d5e4550000, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:43:33,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46403-0x10122d5e4550001, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:43:33,815 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:43:33,816 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T21:43:33,816 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46403-0x10122d5e4550001, quorum=127.0.0.1:62679, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T21:43:33,817 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43251-0x10122d5e4550000, quorum=127.0.0.1:62679, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T21:43:33,817 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T21:43:33,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:43:33,818 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd86ff69d6719,46403,1731188512651' ***** 2024-11-09T21:43:33,818 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-09T21:43:33,818 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-09T21:43:33,818 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-09T21:43:33,819 INFO [RS:0;d86ff69d6719:46403 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-09T21:43:33,819 INFO [RS:0;d86ff69d6719:46403 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-09T21:43:33,819 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(3091): Received CLOSE for be5d9ef20c6a1539c9edfca19c08ac41 2024-11-09T21:43:33,820 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(3091): Received CLOSE for 82505c28df963a1a69ff02aae6cb9d53 2024-11-09T21:43:33,820 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(3091): Received CLOSE for 4d94b8668efb90b9ce664fd703f2c8d3 2024-11-09T21:43:33,820 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(3091): Received CLOSE for 1324ca3e02d44073ea072dea2fead16a 2024-11-09T21:43:33,820 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(3091): Received CLOSE for 0bb8274b2699870e24bf8df1a279f881 2024-11-09T21:43:33,821 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(3091): Received CLOSE for 08ca2f06be250b50389ea4bf06a0b92b 2024-11-09T21:43:33,821 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(3091): Received CLOSE for d2c3f456b99d4783fd5f2aeb363ce6d7 2024-11-09T21:43:33,821 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(3091): Received CLOSE for 20d9071050465d83b5f5ea0aba708a01 2024-11-09T21:43:33,821 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing be5d9ef20c6a1539c9edfca19c08ac41, disabling compactions & flushes 2024-11-09T21:43:33,821 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(3091): Received CLOSE for 16065d24acc6760bfc8ee9d803a26d64 2024-11-09T21:43:33,821 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1731188583312.be5d9ef20c6a1539c9edfca19c08ac41. 2024-11-09T21:43:33,821 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(3091): Received CLOSE for ab051e55337f5a7f78d0c70c373a3851 2024-11-09T21:43:33,821 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1731188583312.be5d9ef20c6a1539c9edfca19c08ac41. 2024-11-09T21:43:33,821 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(959): stopping server d86ff69d6719,46403,1731188512651 2024-11-09T21:43:33,821 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1731188583312.be5d9ef20c6a1539c9edfca19c08ac41. after waiting 0 ms 2024-11-09T21:43:33,821 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1731188583312.be5d9ef20c6a1539c9edfca19c08ac41. 2024-11-09T21:43:33,821 INFO [RS:0;d86ff69d6719:46403 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T21:43:33,821 INFO [RS:0;d86ff69d6719:46403 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d86ff69d6719:46403. 2024-11-09T21:43:33,822 DEBUG [RS:0;d86ff69d6719:46403 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T21:43:33,822 DEBUG [RS:0;d86ff69d6719:46403 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:43:33,822 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-09T21:43:33,822 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-09T21:43:33,822 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-09T21:43:33,822 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-09T21:43:33,823 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-09T21:43:33,823 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-09T21:43:33,823 INFO [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-09T21:43:33,823 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-09T21:43:33,823 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T21:43:33,824 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1351): Waiting on 08ca2f06be250b50389ea4bf06a0b92b, 0bb8274b2699870e24bf8df1a279f881, 1324ca3e02d44073ea072dea2fead16a, 1588230740, 16065d24acc6760bfc8ee9d803a26d64, 20d9071050465d83b5f5ea0aba708a01, 4d94b8668efb90b9ce664fd703f2c8d3, 82505c28df963a1a69ff02aae6cb9d53, ab051e55337f5a7f78d0c70c373a3851, be5d9ef20c6a1539c9edfca19c08ac41, d2c3f456b99d4783fd5f2aeb363ce6d7 2024-11-09T21:43:33,823 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T21:43:33,826 INFO [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-11-09T21:43:33,826 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing be5d9ef20c6a1539c9edfca19c08ac41 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T21:43:33,876 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnNext/be5d9ef20c6a1539c9edfca19c08ac41/.tmp/testFamily/3be7a63c264a4fbcb71e15e4769d23a6 is 61, key is row-0/testFamily:testQualifier/1731188586373/Put/seqid=0 2024-11-09T21:43:33,881 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/meta/1588230740/.tmp/info/56e74be6f43243c4a1898ba832191165 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1731188546641.08ca2f06be250b50389ea4bf06a0b92b./info:regioninfo/1731188547883/Put/seqid=0 2024-11-09T21:43:33,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741855_1031 (size=5049) 2024-11-09T21:43:33,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741856_1032 (size=20564) 2024-11-09T21:43:33,889 INFO [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/meta/1588230740/.tmp/info/56e74be6f43243c4a1898ba832191165 2024-11-09T21:43:33,942 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/meta/1588230740/.tmp/ns/91ff4c1dff2d41d88c3a9cba81145490 is 43, key is default/ns:d/1731188546356/Put/seqid=0 2024-11-09T21:43:33,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741857_1033 (size=5153) 2024-11-09T21:43:34,024 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1351): Waiting on 08ca2f06be250b50389ea4bf06a0b92b, 0bb8274b2699870e24bf8df1a279f881, 1324ca3e02d44073ea072dea2fead16a, 1588230740, 16065d24acc6760bfc8ee9d803a26d64, 20d9071050465d83b5f5ea0aba708a01, 4d94b8668efb90b9ce664fd703f2c8d3, 82505c28df963a1a69ff02aae6cb9d53, ab051e55337f5a7f78d0c70c373a3851, be5d9ef20c6a1539c9edfca19c08ac41, d2c3f456b99d4783fd5f2aeb363ce6d7 2024-11-09T21:43:34,225 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1351): Waiting on 08ca2f06be250b50389ea4bf06a0b92b, 0bb8274b2699870e24bf8df1a279f881, 1324ca3e02d44073ea072dea2fead16a, 1588230740, 16065d24acc6760bfc8ee9d803a26d64, 20d9071050465d83b5f5ea0aba708a01, 4d94b8668efb90b9ce664fd703f2c8d3, 82505c28df963a1a69ff02aae6cb9d53, ab051e55337f5a7f78d0c70c373a3851, be5d9ef20c6a1539c9edfca19c08ac41, d2c3f456b99d4783fd5f2aeb363ce6d7 2024-11-09T21:43:34,260 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-09T21:43:34,286 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnNext/be5d9ef20c6a1539c9edfca19c08ac41/.tmp/testFamily/3be7a63c264a4fbcb71e15e4769d23a6 2024-11-09T21:43:34,302 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnNext/be5d9ef20c6a1539c9edfca19c08ac41/.tmp/testFamily/3be7a63c264a4fbcb71e15e4769d23a6 as hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnNext/be5d9ef20c6a1539c9edfca19c08ac41/testFamily/3be7a63c264a4fbcb71e15e4769d23a6 2024-11-09T21:43:34,311 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnNext/be5d9ef20c6a1539c9edfca19c08ac41/testFamily/3be7a63c264a4fbcb71e15e4769d23a6, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T21:43:34,318 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for be5d9ef20c6a1539c9edfca19c08ac41 in 493ms, sequenceid=8, compaction requested=false 2024-11-09T21:43:34,324 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnNext/be5d9ef20c6a1539c9edfca19c08ac41/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T21:43:34,326 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1731188583312.be5d9ef20c6a1539c9edfca19c08ac41. 2024-11-09T21:43:34,326 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for be5d9ef20c6a1539c9edfca19c08ac41: Waiting for close lock at 1731188613820Running coprocessor pre-close hooks at 1731188613821 (+1 ms)Disabling compacts and flushes for region at 1731188613821Disabling writes for close at 1731188613821Obtaining lock to block concurrent updates at 1731188613826 (+5 ms)Preparing flush snapshotting stores in be5d9ef20c6a1539c9edfca19c08ac41 at 1731188613826Finished memstore snapshotting testNormalScanTimeoutOnNext,,1731188583312.be5d9ef20c6a1539c9edfca19c08ac41., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731188613833 (+7 ms)Flushing stores of testNormalScanTimeoutOnNext,,1731188583312.be5d9ef20c6a1539c9edfca19c08ac41. at 1731188613834 (+1 ms)Flushing be5d9ef20c6a1539c9edfca19c08ac41/testFamily: creating writer at 1731188613835 (+1 ms)Flushing be5d9ef20c6a1539c9edfca19c08ac41/testFamily: appending metadata at 1731188613871 (+36 ms)Flushing be5d9ef20c6a1539c9edfca19c08ac41/testFamily: closing flushed file at 1731188613873 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5224066b: reopening flushed file at 1731188614300 (+427 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for be5d9ef20c6a1539c9edfca19c08ac41 in 493ms, sequenceid=8, compaction requested=false at 1731188614318 (+18 ms)Writing region close event to WAL at 1731188614320 (+2 ms)Running coprocessor post-close hooks at 1731188614325 (+5 ms)Closed at 1731188614326 (+1 ms) 2024-11-09T21:43:34,327 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1731188583312.be5d9ef20c6a1539c9edfca19c08ac41. 2024-11-09T21:43:34,327 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 82505c28df963a1a69ff02aae6cb9d53, disabling compactions & flushes 2024-11-09T21:43:34,327 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731188589413.82505c28df963a1a69ff02aae6cb9d53. 2024-11-09T21:43:34,327 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731188589413.82505c28df963a1a69ff02aae6cb9d53. 2024-11-09T21:43:34,327 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731188589413.82505c28df963a1a69ff02aae6cb9d53. after waiting 0 ms 2024-11-09T21:43:34,327 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731188589413.82505c28df963a1a69ff02aae6cb9d53. 2024-11-09T21:43:34,327 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 82505c28df963a1a69ff02aae6cb9d53 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T21:43:34,327 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T21:43:34,347 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/82505c28df963a1a69ff02aae6cb9d53/.tmp/testFamily/462e823b38354a9aa7b068aef7e5b99a is 61, key is row-0/testFamily:testQualifier/1731188592475/Put/seqid=0 2024-11-09T21:43:34,348 INFO [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/meta/1588230740/.tmp/ns/91ff4c1dff2d41d88c3a9cba81145490 2024-11-09T21:43:34,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741858_1034 (size=5049) 2024-11-09T21:43:34,374 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/meta/1588230740/.tmp/table/c714600ecddd4601b6c129d426d220ce is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1731188547900/Put/seqid=0 2024-11-09T21:43:34,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741859_1035 (size=6699) 2024-11-09T21:43:34,425 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1351): Waiting on 08ca2f06be250b50389ea4bf06a0b92b, 0bb8274b2699870e24bf8df1a279f881, 1324ca3e02d44073ea072dea2fead16a, 1588230740, 16065d24acc6760bfc8ee9d803a26d64, 20d9071050465d83b5f5ea0aba708a01, 4d94b8668efb90b9ce664fd703f2c8d3, 82505c28df963a1a69ff02aae6cb9d53, ab051e55337f5a7f78d0c70c373a3851, d2c3f456b99d4783fd5f2aeb363ce6d7 2024-11-09T21:43:34,433 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=d86ff69d6719:43251 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:?] ... 10 more 2024-11-09T21:43:34,613 INFO [regionserver/d86ff69d6719:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-09T21:43:34,613 INFO [regionserver/d86ff69d6719:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-09T21:43:34,625 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1351): Waiting on 08ca2f06be250b50389ea4bf06a0b92b, 0bb8274b2699870e24bf8df1a279f881, 1324ca3e02d44073ea072dea2fead16a, 1588230740, 16065d24acc6760bfc8ee9d803a26d64, 20d9071050465d83b5f5ea0aba708a01, 4d94b8668efb90b9ce664fd703f2c8d3, 82505c28df963a1a69ff02aae6cb9d53, ab051e55337f5a7f78d0c70c373a3851, d2c3f456b99d4783fd5f2aeb363ce6d7 2024-11-09T21:43:34,755 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/82505c28df963a1a69ff02aae6cb9d53/.tmp/testFamily/462e823b38354a9aa7b068aef7e5b99a 2024-11-09T21:43:34,771 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/82505c28df963a1a69ff02aae6cb9d53/.tmp/testFamily/462e823b38354a9aa7b068aef7e5b99a as hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/82505c28df963a1a69ff02aae6cb9d53/testFamily/462e823b38354a9aa7b068aef7e5b99a 2024-11-09T21:43:34,779 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/82505c28df963a1a69ff02aae6cb9d53/testFamily/462e823b38354a9aa7b068aef7e5b99a, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T21:43:34,780 INFO [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/meta/1588230740/.tmp/table/c714600ecddd4601b6c129d426d220ce 2024-11-09T21:43:34,781 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 82505c28df963a1a69ff02aae6cb9d53 in 454ms, sequenceid=8, compaction requested=false 2024-11-09T21:43:34,785 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/82505c28df963a1a69ff02aae6cb9d53/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T21:43:34,786 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731188589413.82505c28df963a1a69ff02aae6cb9d53. 2024-11-09T21:43:34,786 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 82505c28df963a1a69ff02aae6cb9d53: Waiting for close lock at 1731188614327Running coprocessor pre-close hooks at 1731188614327Disabling compacts and flushes for region at 1731188614327Disabling writes for close at 1731188614327Obtaining lock to block concurrent updates at 1731188614327Preparing flush snapshotting stores in 82505c28df963a1a69ff02aae6cb9d53 at 1731188614327Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731188589413.82505c28df963a1a69ff02aae6cb9d53., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731188614327Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731188589413.82505c28df963a1a69ff02aae6cb9d53. at 1731188614328 (+1 ms)Flushing 82505c28df963a1a69ff02aae6cb9d53/testFamily: creating writer at 1731188614328Flushing 82505c28df963a1a69ff02aae6cb9d53/testFamily: appending metadata at 1731188614347 (+19 ms)Flushing 82505c28df963a1a69ff02aae6cb9d53/testFamily: closing flushed file at 1731188614347Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a5fc286: reopening flushed file at 1731188614770 (+423 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 82505c28df963a1a69ff02aae6cb9d53 in 454ms, sequenceid=8, compaction requested=false at 1731188614781 (+11 ms)Writing region close event to WAL at 1731188614781Running coprocessor post-close hooks at 1731188614786 (+5 ms)Closed at 1731188614786 2024-11-09T21:43:34,786 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1731188589413.82505c28df963a1a69ff02aae6cb9d53. 2024-11-09T21:43:34,787 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4d94b8668efb90b9ce664fd703f2c8d3, disabling compactions & flushes 2024-11-09T21:43:34,787 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1731188598524.4d94b8668efb90b9ce664fd703f2c8d3. 2024-11-09T21:43:34,787 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1731188598524.4d94b8668efb90b9ce664fd703f2c8d3. 2024-11-09T21:43:34,787 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1731188598524.4d94b8668efb90b9ce664fd703f2c8d3. after waiting 0 ms 2024-11-09T21:43:34,787 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1731188598524.4d94b8668efb90b9ce664fd703f2c8d3. 2024-11-09T21:43:34,787 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 4d94b8668efb90b9ce664fd703f2c8d3 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T21:43:34,790 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/meta/1588230740/.tmp/info/56e74be6f43243c4a1898ba832191165 as hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/meta/1588230740/info/56e74be6f43243c4a1898ba832191165 2024-11-09T21:43:34,799 INFO [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/meta/1588230740/info/56e74be6f43243c4a1898ba832191165, entries=100, sequenceid=56, filesize=20.1 K 2024-11-09T21:43:34,801 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/meta/1588230740/.tmp/ns/91ff4c1dff2d41d88c3a9cba81145490 as hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/meta/1588230740/ns/91ff4c1dff2d41d88c3a9cba81145490 2024-11-09T21:43:34,805 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnNext/4d94b8668efb90b9ce664fd703f2c8d3/.tmp/testFamily/bc34a201fbd34c16bd646d9a208401c4 is 61, key is row-0/testFamily:testQualifier/1731188601575/Put/seqid=0 2024-11-09T21:43:34,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741860_1036 (size=5049) 2024-11-09T21:43:34,810 INFO [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/meta/1588230740/ns/91ff4c1dff2d41d88c3a9cba81145490, entries=2, sequenceid=56, filesize=5.0 K 2024-11-09T21:43:34,811 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/meta/1588230740/.tmp/table/c714600ecddd4601b6c129d426d220ce as hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/meta/1588230740/table/c714600ecddd4601b6c129d426d220ce 2024-11-09T21:43:34,819 INFO [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/meta/1588230740/table/c714600ecddd4601b6c129d426d220ce, entries=20, sequenceid=56, filesize=6.5 K 2024-11-09T21:43:34,820 INFO [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 995ms, sequenceid=56, compaction requested=false 2024-11-09T21:43:34,825 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-11-09T21:43:34,825 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1321): Waiting on 9 regions to close 2024-11-09T21:43:34,826 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1325): Online Regions={4d94b8668efb90b9ce664fd703f2c8d3=hbase:testMetaScanTimeoutOnNext,,1731188598524.4d94b8668efb90b9ce664fd703f2c8d3., 1324ca3e02d44073ea072dea2fead16a=testNormalScanTimeoutOnNextAsync,,1731188577216.1324ca3e02d44073ea072dea2fead16a., 1588230740=hbase:meta,,1.1588230740, 0bb8274b2699870e24bf8df1a279f881=hbase:testMetaScanTimeoutOnNextAsync,,1731188558918.0bb8274b2699870e24bf8df1a279f881., 08ca2f06be250b50389ea4bf06a0b92b=testRetryOutOfOrderScannerNextExceptionAsync,,1731188546641.08ca2f06be250b50389ea4bf06a0b92b., d2c3f456b99d4783fd5f2aeb363ce6d7=testRetryOutOfOrderScannerNextException,,1731188568032.d2c3f456b99d4783fd5f2aeb363ce6d7., 20d9071050465d83b5f5ea0aba708a01=hbase:testMetaScanTimeoutOnOpenScanner,,1731188549820.20d9071050465d83b5f5ea0aba708a01., 16065d24acc6760bfc8ee9d803a26d64=testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64., ab051e55337f5a7f78d0c70c373a3851=testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851.} 2024-11-09T21:43:34,826 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1351): Waiting on 08ca2f06be250b50389ea4bf06a0b92b, 0bb8274b2699870e24bf8df1a279f881, 1324ca3e02d44073ea072dea2fead16a, 1588230740, 16065d24acc6760bfc8ee9d803a26d64, 20d9071050465d83b5f5ea0aba708a01, 4d94b8668efb90b9ce664fd703f2c8d3, ab051e55337f5a7f78d0c70c373a3851, d2c3f456b99d4783fd5f2aeb363ce6d7 2024-11-09T21:43:34,826 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T21:43:34,826 INFO [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-09T21:43:34,827 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731188613823Running coprocessor pre-close hooks at 1731188613823Disabling compacts and flushes for region at 1731188613823Disabling writes for close at 1731188613823Obtaining lock to block concurrent updates at 1731188613826 (+3 ms)Preparing flush snapshotting stores in 1588230740 at 1731188613826Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1731188613833 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731188613834 (+1 ms)Flushing 1588230740/info: creating writer at 1731188613835 (+1 ms)Flushing 1588230740/info: appending metadata at 1731188613877 (+42 ms)Flushing 1588230740/info: closing flushed file at 1731188613877Flushing 1588230740/ns: creating writer at 1731188613926 (+49 ms)Flushing 1588230740/ns: appending metadata at 1731188613941 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731188613941Flushing 1588230740/table: creating writer at 1731188614357 (+416 ms)Flushing 1588230740/table: appending metadata at 1731188614374 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731188614374Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63b067b9: reopening flushed file at 1731188614789 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72a67924: reopening flushed file at 1731188614799 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63d0f069: reopening flushed file at 1731188614810 (+11 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 995ms, sequenceid=56, compaction requested=false at 1731188614820 (+10 ms)Writing region close event to WAL at 1731188614821 (+1 ms)Running coprocessor post-close hooks at 1731188614826 (+5 ms)Closed at 1731188614826 2024-11-09T21:43:34,827 DEBUG [RS_CLOSE_META-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-09T21:43:35,026 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1351): Waiting on 08ca2f06be250b50389ea4bf06a0b92b, 0bb8274b2699870e24bf8df1a279f881, 1324ca3e02d44073ea072dea2fead16a, 16065d24acc6760bfc8ee9d803a26d64, 20d9071050465d83b5f5ea0aba708a01, 4d94b8668efb90b9ce664fd703f2c8d3, ab051e55337f5a7f78d0c70c373a3851, d2c3f456b99d4783fd5f2aeb363ce6d7 2024-11-09T21:43:35,213 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnNext/4d94b8668efb90b9ce664fd703f2c8d3/.tmp/testFamily/bc34a201fbd34c16bd646d9a208401c4 2024-11-09T21:43:35,226 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1351): Waiting on 08ca2f06be250b50389ea4bf06a0b92b, 0bb8274b2699870e24bf8df1a279f881, 1324ca3e02d44073ea072dea2fead16a, 16065d24acc6760bfc8ee9d803a26d64, 20d9071050465d83b5f5ea0aba708a01, 4d94b8668efb90b9ce664fd703f2c8d3, ab051e55337f5a7f78d0c70c373a3851, d2c3f456b99d4783fd5f2aeb363ce6d7 2024-11-09T21:43:35,227 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnNext/4d94b8668efb90b9ce664fd703f2c8d3/.tmp/testFamily/bc34a201fbd34c16bd646d9a208401c4 as hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnNext/4d94b8668efb90b9ce664fd703f2c8d3/testFamily/bc34a201fbd34c16bd646d9a208401c4 2024-11-09T21:43:35,236 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnNext/4d94b8668efb90b9ce664fd703f2c8d3/testFamily/bc34a201fbd34c16bd646d9a208401c4, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T21:43:35,237 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4d94b8668efb90b9ce664fd703f2c8d3 in 450ms, sequenceid=8, compaction requested=false 2024-11-09T21:43:35,243 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnNext/4d94b8668efb90b9ce664fd703f2c8d3/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T21:43:35,244 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1731188598524.4d94b8668efb90b9ce664fd703f2c8d3. 2024-11-09T21:43:35,244 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4d94b8668efb90b9ce664fd703f2c8d3: Waiting for close lock at 1731188614787Running coprocessor pre-close hooks at 1731188614787Disabling compacts and flushes for region at 1731188614787Disabling writes for close at 1731188614787Obtaining lock to block concurrent updates at 1731188614787Preparing flush snapshotting stores in 4d94b8668efb90b9ce664fd703f2c8d3 at 1731188614787Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1731188598524.4d94b8668efb90b9ce664fd703f2c8d3., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731188614787Flushing stores of hbase:testMetaScanTimeoutOnNext,,1731188598524.4d94b8668efb90b9ce664fd703f2c8d3. at 1731188614788 (+1 ms)Flushing 4d94b8668efb90b9ce664fd703f2c8d3/testFamily: creating writer at 1731188614788Flushing 4d94b8668efb90b9ce664fd703f2c8d3/testFamily: appending metadata at 1731188614805 (+17 ms)Flushing 4d94b8668efb90b9ce664fd703f2c8d3/testFamily: closing flushed file at 1731188614805Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f93d602: reopening flushed file at 1731188615226 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 4d94b8668efb90b9ce664fd703f2c8d3 in 450ms, sequenceid=8, compaction requested=false at 1731188615237 (+11 ms)Writing region close event to WAL at 1731188615238 (+1 ms)Running coprocessor post-close hooks at 1731188615244 (+6 ms)Closed at 1731188615244 2024-11-09T21:43:35,245 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1731188598524.4d94b8668efb90b9ce664fd703f2c8d3. 2024-11-09T21:43:35,245 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1324ca3e02d44073ea072dea2fead16a, disabling compactions & flushes 2024-11-09T21:43:35,245 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1731188577216.1324ca3e02d44073ea072dea2fead16a. 2024-11-09T21:43:35,245 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1731188577216.1324ca3e02d44073ea072dea2fead16a. 2024-11-09T21:43:35,245 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1731188577216.1324ca3e02d44073ea072dea2fead16a. after waiting 0 ms 2024-11-09T21:43:35,245 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1731188577216.1324ca3e02d44073ea072dea2fead16a. 2024-11-09T21:43:35,245 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 1324ca3e02d44073ea072dea2fead16a 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T21:43:35,263 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnNextAsync/1324ca3e02d44073ea072dea2fead16a/.tmp/testFamily/85a6bc1d095846c1a488b9b4ad2f047b is 61, key is row-0/testFamily:testQualifier/1731188580267/Put/seqid=0 2024-11-09T21:43:35,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741861_1037 (size=5049) 2024-11-09T21:43:35,427 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1351): Waiting on 08ca2f06be250b50389ea4bf06a0b92b, 0bb8274b2699870e24bf8df1a279f881, 1324ca3e02d44073ea072dea2fead16a, 16065d24acc6760bfc8ee9d803a26d64, 20d9071050465d83b5f5ea0aba708a01, ab051e55337f5a7f78d0c70c373a3851, d2c3f456b99d4783fd5f2aeb363ce6d7 2024-11-09T21:43:35,627 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1351): Waiting on 08ca2f06be250b50389ea4bf06a0b92b, 0bb8274b2699870e24bf8df1a279f881, 1324ca3e02d44073ea072dea2fead16a, 16065d24acc6760bfc8ee9d803a26d64, 20d9071050465d83b5f5ea0aba708a01, ab051e55337f5a7f78d0c70c373a3851, d2c3f456b99d4783fd5f2aeb363ce6d7 2024-11-09T21:43:35,671 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnNextAsync/1324ca3e02d44073ea072dea2fead16a/.tmp/testFamily/85a6bc1d095846c1a488b9b4ad2f047b 2024-11-09T21:43:35,685 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnNextAsync/1324ca3e02d44073ea072dea2fead16a/.tmp/testFamily/85a6bc1d095846c1a488b9b4ad2f047b as hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnNextAsync/1324ca3e02d44073ea072dea2fead16a/testFamily/85a6bc1d095846c1a488b9b4ad2f047b 2024-11-09T21:43:35,692 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnNextAsync/1324ca3e02d44073ea072dea2fead16a/testFamily/85a6bc1d095846c1a488b9b4ad2f047b, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T21:43:35,693 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1324ca3e02d44073ea072dea2fead16a in 448ms, sequenceid=8, compaction requested=false 2024-11-09T21:43:35,699 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnNextAsync/1324ca3e02d44073ea072dea2fead16a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T21:43:35,700 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1731188577216.1324ca3e02d44073ea072dea2fead16a. 2024-11-09T21:43:35,700 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1324ca3e02d44073ea072dea2fead16a: Waiting for close lock at 1731188615245Running coprocessor pre-close hooks at 1731188615245Disabling compacts and flushes for region at 1731188615245Disabling writes for close at 1731188615245Obtaining lock to block concurrent updates at 1731188615245Preparing flush snapshotting stores in 1324ca3e02d44073ea072dea2fead16a at 1731188615245Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1731188577216.1324ca3e02d44073ea072dea2fead16a., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731188615245Flushing stores of testNormalScanTimeoutOnNextAsync,,1731188577216.1324ca3e02d44073ea072dea2fead16a. at 1731188615246 (+1 ms)Flushing 1324ca3e02d44073ea072dea2fead16a/testFamily: creating writer at 1731188615246Flushing 1324ca3e02d44073ea072dea2fead16a/testFamily: appending metadata at 1731188615263 (+17 ms)Flushing 1324ca3e02d44073ea072dea2fead16a/testFamily: closing flushed file at 1731188615263Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c575989: reopening flushed file at 1731188615684 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 1324ca3e02d44073ea072dea2fead16a in 448ms, sequenceid=8, compaction requested=false at 1731188615693 (+9 ms)Writing region close event to WAL at 1731188615694 (+1 ms)Running coprocessor post-close hooks at 1731188615700 (+6 ms)Closed at 1731188615700 2024-11-09T21:43:35,700 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1731188577216.1324ca3e02d44073ea072dea2fead16a. 2024-11-09T21:43:35,700 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0bb8274b2699870e24bf8df1a279f881, disabling compactions & flushes 2024-11-09T21:43:35,700 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1731188558918.0bb8274b2699870e24bf8df1a279f881. 2024-11-09T21:43:35,700 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731188558918.0bb8274b2699870e24bf8df1a279f881. 2024-11-09T21:43:35,700 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1731188558918.0bb8274b2699870e24bf8df1a279f881. after waiting 0 ms 2024-11-09T21:43:35,700 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1731188558918.0bb8274b2699870e24bf8df1a279f881. 2024-11-09T21:43:35,700 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 0bb8274b2699870e24bf8df1a279f881 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T21:43:35,717 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnNextAsync/0bb8274b2699870e24bf8df1a279f881/.tmp/testFamily/a58d794754c24539ba9842660a850f8a is 61, key is row-0/testFamily:testQualifier/1731188561983/Put/seqid=0 2024-11-09T21:43:35,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741862_1038 (size=5049) 2024-11-09T21:43:35,827 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1321): Waiting on 6 regions to close 2024-11-09T21:43:35,828 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1325): Online Regions={0bb8274b2699870e24bf8df1a279f881=hbase:testMetaScanTimeoutOnNextAsync,,1731188558918.0bb8274b2699870e24bf8df1a279f881., 08ca2f06be250b50389ea4bf06a0b92b=testRetryOutOfOrderScannerNextExceptionAsync,,1731188546641.08ca2f06be250b50389ea4bf06a0b92b., d2c3f456b99d4783fd5f2aeb363ce6d7=testRetryOutOfOrderScannerNextException,,1731188568032.d2c3f456b99d4783fd5f2aeb363ce6d7., 20d9071050465d83b5f5ea0aba708a01=hbase:testMetaScanTimeoutOnOpenScanner,,1731188549820.20d9071050465d83b5f5ea0aba708a01., 16065d24acc6760bfc8ee9d803a26d64=testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64., ab051e55337f5a7f78d0c70c373a3851=testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851.} 2024-11-09T21:43:35,828 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1351): Waiting on 08ca2f06be250b50389ea4bf06a0b92b, 0bb8274b2699870e24bf8df1a279f881, 16065d24acc6760bfc8ee9d803a26d64, 20d9071050465d83b5f5ea0aba708a01, ab051e55337f5a7f78d0c70c373a3851, d2c3f456b99d4783fd5f2aeb363ce6d7 2024-11-09T21:43:36,028 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1351): Waiting on 08ca2f06be250b50389ea4bf06a0b92b, 0bb8274b2699870e24bf8df1a279f881, 16065d24acc6760bfc8ee9d803a26d64, 20d9071050465d83b5f5ea0aba708a01, ab051e55337f5a7f78d0c70c373a3851, d2c3f456b99d4783fd5f2aeb363ce6d7 2024-11-09T21:43:36,124 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnNextAsync/0bb8274b2699870e24bf8df1a279f881/.tmp/testFamily/a58d794754c24539ba9842660a850f8a 2024-11-09T21:43:36,138 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnNextAsync/0bb8274b2699870e24bf8df1a279f881/.tmp/testFamily/a58d794754c24539ba9842660a850f8a as hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnNextAsync/0bb8274b2699870e24bf8df1a279f881/testFamily/a58d794754c24539ba9842660a850f8a 2024-11-09T21:43:36,146 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnNextAsync/0bb8274b2699870e24bf8df1a279f881/testFamily/a58d794754c24539ba9842660a850f8a, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T21:43:36,147 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0bb8274b2699870e24bf8df1a279f881 in 447ms, sequenceid=8, compaction requested=false 2024-11-09T21:43:36,152 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnNextAsync/0bb8274b2699870e24bf8df1a279f881/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T21:43:36,152 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1731188558918.0bb8274b2699870e24bf8df1a279f881. 2024-11-09T21:43:36,153 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0bb8274b2699870e24bf8df1a279f881: Waiting for close lock at 1731188615700Running coprocessor pre-close hooks at 1731188615700Disabling compacts and flushes for region at 1731188615700Disabling writes for close at 1731188615700Obtaining lock to block concurrent updates at 1731188615700Preparing flush snapshotting stores in 0bb8274b2699870e24bf8df1a279f881 at 1731188615700Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1731188558918.0bb8274b2699870e24bf8df1a279f881., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731188615701 (+1 ms)Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1731188558918.0bb8274b2699870e24bf8df1a279f881. at 1731188615701Flushing 0bb8274b2699870e24bf8df1a279f881/testFamily: creating writer at 1731188615702 (+1 ms)Flushing 0bb8274b2699870e24bf8df1a279f881/testFamily: appending metadata at 1731188615717 (+15 ms)Flushing 0bb8274b2699870e24bf8df1a279f881/testFamily: closing flushed file at 1731188615717Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48d9442d: reopening flushed file at 1731188616137 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0bb8274b2699870e24bf8df1a279f881 in 447ms, sequenceid=8, compaction requested=false at 1731188616147 (+10 ms)Writing region close event to WAL at 1731188616148 (+1 ms)Running coprocessor post-close hooks at 1731188616152 (+4 ms)Closed at 1731188616152 2024-11-09T21:43:36,153 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1731188558918.0bb8274b2699870e24bf8df1a279f881. 2024-11-09T21:43:36,153 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 08ca2f06be250b50389ea4bf06a0b92b, disabling compactions & flushes 2024-11-09T21:43:36,153 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1731188546641.08ca2f06be250b50389ea4bf06a0b92b. 2024-11-09T21:43:36,153 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731188546641.08ca2f06be250b50389ea4bf06a0b92b. 2024-11-09T21:43:36,153 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1731188546641.08ca2f06be250b50389ea4bf06a0b92b. after waiting 0 ms 2024-11-09T21:43:36,153 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1731188546641.08ca2f06be250b50389ea4bf06a0b92b. 2024-11-09T21:43:36,153 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 08ca2f06be250b50389ea4bf06a0b92b 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T21:43:36,170 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testRetryOutOfOrderScannerNextExceptionAsync/08ca2f06be250b50389ea4bf06a0b92b/.tmp/testFamily/d560fc8b587b4205a4faac25743e8c55 is 61, key is row-0/testFamily:testQualifier/1731188549727/Put/seqid=0 2024-11-09T21:43:36,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741863_1039 (size=5049) 2024-11-09T21:43:36,228 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1351): Waiting on 08ca2f06be250b50389ea4bf06a0b92b, 16065d24acc6760bfc8ee9d803a26d64, 20d9071050465d83b5f5ea0aba708a01, ab051e55337f5a7f78d0c70c373a3851, d2c3f456b99d4783fd5f2aeb363ce6d7 2024-11-09T21:43:36,429 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1351): Waiting on 08ca2f06be250b50389ea4bf06a0b92b, 16065d24acc6760bfc8ee9d803a26d64, 20d9071050465d83b5f5ea0aba708a01, ab051e55337f5a7f78d0c70c373a3851, d2c3f456b99d4783fd5f2aeb363ce6d7 2024-11-09T21:43:36,577 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testRetryOutOfOrderScannerNextExceptionAsync/08ca2f06be250b50389ea4bf06a0b92b/.tmp/testFamily/d560fc8b587b4205a4faac25743e8c55 2024-11-09T21:43:36,592 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testRetryOutOfOrderScannerNextExceptionAsync/08ca2f06be250b50389ea4bf06a0b92b/.tmp/testFamily/d560fc8b587b4205a4faac25743e8c55 as hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testRetryOutOfOrderScannerNextExceptionAsync/08ca2f06be250b50389ea4bf06a0b92b/testFamily/d560fc8b587b4205a4faac25743e8c55 2024-11-09T21:43:36,602 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testRetryOutOfOrderScannerNextExceptionAsync/08ca2f06be250b50389ea4bf06a0b92b/testFamily/d560fc8b587b4205a4faac25743e8c55, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T21:43:36,603 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 08ca2f06be250b50389ea4bf06a0b92b in 450ms, sequenceid=8, compaction requested=false 2024-11-09T21:43:36,608 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testRetryOutOfOrderScannerNextExceptionAsync/08ca2f06be250b50389ea4bf06a0b92b/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T21:43:36,608 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1731188546641.08ca2f06be250b50389ea4bf06a0b92b. 2024-11-09T21:43:36,609 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 08ca2f06be250b50389ea4bf06a0b92b: Waiting for close lock at 1731188616153Running coprocessor pre-close hooks at 1731188616153Disabling compacts and flushes for region at 1731188616153Disabling writes for close at 1731188616153Obtaining lock to block concurrent updates at 1731188616153Preparing flush snapshotting stores in 08ca2f06be250b50389ea4bf06a0b92b at 1731188616153Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1731188546641.08ca2f06be250b50389ea4bf06a0b92b., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731188616154 (+1 ms)Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1731188546641.08ca2f06be250b50389ea4bf06a0b92b. at 1731188616154Flushing 08ca2f06be250b50389ea4bf06a0b92b/testFamily: creating writer at 1731188616155 (+1 ms)Flushing 08ca2f06be250b50389ea4bf06a0b92b/testFamily: appending metadata at 1731188616170 (+15 ms)Flushing 08ca2f06be250b50389ea4bf06a0b92b/testFamily: closing flushed file at 1731188616170Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1972ef31: reopening flushed file at 1731188616590 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 08ca2f06be250b50389ea4bf06a0b92b in 450ms, sequenceid=8, compaction requested=false at 1731188616603 (+13 ms)Writing region close event to WAL at 1731188616604 (+1 ms)Running coprocessor post-close hooks at 1731188616608 (+4 ms)Closed at 1731188616608 2024-11-09T21:43:36,609 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1731188546641.08ca2f06be250b50389ea4bf06a0b92b. 2024-11-09T21:43:36,609 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d2c3f456b99d4783fd5f2aeb363ce6d7, disabling compactions & flushes 2024-11-09T21:43:36,609 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1731188568032.d2c3f456b99d4783fd5f2aeb363ce6d7. 2024-11-09T21:43:36,609 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1731188568032.d2c3f456b99d4783fd5f2aeb363ce6d7. 2024-11-09T21:43:36,609 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1731188568032.d2c3f456b99d4783fd5f2aeb363ce6d7. after waiting 0 ms 2024-11-09T21:43:36,609 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1731188568032.d2c3f456b99d4783fd5f2aeb363ce6d7. 2024-11-09T21:43:36,609 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing d2c3f456b99d4783fd5f2aeb363ce6d7 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T21:43:36,625 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testRetryOutOfOrderScannerNextException/d2c3f456b99d4783fd5f2aeb363ce6d7/.tmp/testFamily/14a3df165da048f0a11a62f51715f0a1 is 61, key is row-0/testFamily:testQualifier/1731188571072/Put/seqid=0 2024-11-09T21:43:36,629 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1351): Waiting on 16065d24acc6760bfc8ee9d803a26d64, 20d9071050465d83b5f5ea0aba708a01, ab051e55337f5a7f78d0c70c373a3851, d2c3f456b99d4783fd5f2aeb363ce6d7 2024-11-09T21:43:36,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741864_1040 (size=5049) 2024-11-09T21:43:36,829 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1321): Waiting on 4 regions to close 2024-11-09T21:43:36,830 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1325): Online Regions={d2c3f456b99d4783fd5f2aeb363ce6d7=testRetryOutOfOrderScannerNextException,,1731188568032.d2c3f456b99d4783fd5f2aeb363ce6d7., 20d9071050465d83b5f5ea0aba708a01=hbase:testMetaScanTimeoutOnOpenScanner,,1731188549820.20d9071050465d83b5f5ea0aba708a01., 16065d24acc6760bfc8ee9d803a26d64=testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64., ab051e55337f5a7f78d0c70c373a3851=testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851.} 2024-11-09T21:43:36,830 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1351): Waiting on 16065d24acc6760bfc8ee9d803a26d64, 20d9071050465d83b5f5ea0aba708a01, ab051e55337f5a7f78d0c70c373a3851, d2c3f456b99d4783fd5f2aeb363ce6d7 2024-11-09T21:43:37,030 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1351): Waiting on 16065d24acc6760bfc8ee9d803a26d64, 20d9071050465d83b5f5ea0aba708a01, ab051e55337f5a7f78d0c70c373a3851, d2c3f456b99d4783fd5f2aeb363ce6d7 2024-11-09T21:43:37,032 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testRetryOutOfOrderScannerNextException/d2c3f456b99d4783fd5f2aeb363ce6d7/.tmp/testFamily/14a3df165da048f0a11a62f51715f0a1 2024-11-09T21:43:37,046 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testRetryOutOfOrderScannerNextException/d2c3f456b99d4783fd5f2aeb363ce6d7/.tmp/testFamily/14a3df165da048f0a11a62f51715f0a1 as hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testRetryOutOfOrderScannerNextException/d2c3f456b99d4783fd5f2aeb363ce6d7/testFamily/14a3df165da048f0a11a62f51715f0a1 2024-11-09T21:43:37,052 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testRetryOutOfOrderScannerNextException/d2c3f456b99d4783fd5f2aeb363ce6d7/testFamily/14a3df165da048f0a11a62f51715f0a1, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T21:43:37,054 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d2c3f456b99d4783fd5f2aeb363ce6d7 in 444ms, sequenceid=8, compaction requested=false 2024-11-09T21:43:37,058 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testRetryOutOfOrderScannerNextException/d2c3f456b99d4783fd5f2aeb363ce6d7/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T21:43:37,059 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1731188568032.d2c3f456b99d4783fd5f2aeb363ce6d7. 2024-11-09T21:43:37,059 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d2c3f456b99d4783fd5f2aeb363ce6d7: Waiting for close lock at 1731188616609Running coprocessor pre-close hooks at 1731188616609Disabling compacts and flushes for region at 1731188616609Disabling writes for close at 1731188616609Obtaining lock to block concurrent updates at 1731188616609Preparing flush snapshotting stores in d2c3f456b99d4783fd5f2aeb363ce6d7 at 1731188616609Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1731188568032.d2c3f456b99d4783fd5f2aeb363ce6d7., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731188616609Flushing stores of testRetryOutOfOrderScannerNextException,,1731188568032.d2c3f456b99d4783fd5f2aeb363ce6d7. at 1731188616610 (+1 ms)Flushing d2c3f456b99d4783fd5f2aeb363ce6d7/testFamily: creating writer at 1731188616610Flushing d2c3f456b99d4783fd5f2aeb363ce6d7/testFamily: appending metadata at 1731188616625 (+15 ms)Flushing d2c3f456b99d4783fd5f2aeb363ce6d7/testFamily: closing flushed file at 1731188616625Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59e062bb: reopening flushed file at 1731188617045 (+420 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for d2c3f456b99d4783fd5f2aeb363ce6d7 in 444ms, sequenceid=8, compaction requested=false at 1731188617054 (+9 ms)Writing region close event to WAL at 1731188617054Running coprocessor post-close hooks at 1731188617059 (+5 ms)Closed at 1731188617059 2024-11-09T21:43:37,059 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1731188568032.d2c3f456b99d4783fd5f2aeb363ce6d7. 2024-11-09T21:43:37,059 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 20d9071050465d83b5f5ea0aba708a01, disabling compactions & flushes 2024-11-09T21:43:37,059 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1731188549820.20d9071050465d83b5f5ea0aba708a01. 2024-11-09T21:43:37,059 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731188549820.20d9071050465d83b5f5ea0aba708a01. 2024-11-09T21:43:37,060 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1731188549820.20d9071050465d83b5f5ea0aba708a01. after waiting 0 ms 2024-11-09T21:43:37,060 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1731188549820.20d9071050465d83b5f5ea0aba708a01. 2024-11-09T21:43:37,060 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 20d9071050465d83b5f5ea0aba708a01 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T21:43:37,075 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnOpenScanner/20d9071050465d83b5f5ea0aba708a01/.tmp/testFamily/a37a7729424e4b7ca0abd3161731d35d is 61, key is row-0/testFamily:testQualifier/1731188552863/Put/seqid=0 2024-11-09T21:43:37,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741865_1041 (size=5049) 2024-11-09T21:43:37,231 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1351): Waiting on 16065d24acc6760bfc8ee9d803a26d64, 20d9071050465d83b5f5ea0aba708a01, ab051e55337f5a7f78d0c70c373a3851 2024-11-09T21:43:37,431 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1351): Waiting on 16065d24acc6760bfc8ee9d803a26d64, 20d9071050465d83b5f5ea0aba708a01, ab051e55337f5a7f78d0c70c373a3851 2024-11-09T21:43:37,482 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnOpenScanner/20d9071050465d83b5f5ea0aba708a01/.tmp/testFamily/a37a7729424e4b7ca0abd3161731d35d 2024-11-09T21:43:37,495 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnOpenScanner/20d9071050465d83b5f5ea0aba708a01/.tmp/testFamily/a37a7729424e4b7ca0abd3161731d35d as hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnOpenScanner/20d9071050465d83b5f5ea0aba708a01/testFamily/a37a7729424e4b7ca0abd3161731d35d 2024-11-09T21:43:37,502 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnOpenScanner/20d9071050465d83b5f5ea0aba708a01/testFamily/a37a7729424e4b7ca0abd3161731d35d, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T21:43:37,503 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 20d9071050465d83b5f5ea0aba708a01 in 443ms, sequenceid=8, compaction requested=false 2024-11-09T21:43:37,508 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/hbase/testMetaScanTimeoutOnOpenScanner/20d9071050465d83b5f5ea0aba708a01/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T21:43:37,509 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1731188549820.20d9071050465d83b5f5ea0aba708a01. 2024-11-09T21:43:37,509 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 20d9071050465d83b5f5ea0aba708a01: Waiting for close lock at 1731188617059Running coprocessor pre-close hooks at 1731188617059Disabling compacts and flushes for region at 1731188617059Disabling writes for close at 1731188617060 (+1 ms)Obtaining lock to block concurrent updates at 1731188617060Preparing flush snapshotting stores in 20d9071050465d83b5f5ea0aba708a01 at 1731188617060Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1731188549820.20d9071050465d83b5f5ea0aba708a01., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731188617060Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1731188549820.20d9071050465d83b5f5ea0aba708a01. at 1731188617060Flushing 20d9071050465d83b5f5ea0aba708a01/testFamily: creating writer at 1731188617061 (+1 ms)Flushing 20d9071050465d83b5f5ea0aba708a01/testFamily: appending metadata at 1731188617075 (+14 ms)Flushing 20d9071050465d83b5f5ea0aba708a01/testFamily: closing flushed file at 1731188617075Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@158d855f: reopening flushed file at 1731188617493 (+418 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 20d9071050465d83b5f5ea0aba708a01 in 443ms, sequenceid=8, compaction requested=false at 1731188617503 (+10 ms)Writing region close event to WAL at 1731188617504 (+1 ms)Running coprocessor post-close hooks at 1731188617509 (+5 ms)Closed at 1731188617509 2024-11-09T21:43:37,509 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1731188549820.20d9071050465d83b5f5ea0aba708a01. 2024-11-09T21:43:37,509 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 16065d24acc6760bfc8ee9d803a26d64, disabling compactions & flushes 2024-11-09T21:43:37,509 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64. 2024-11-09T21:43:37,509 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64. 2024-11-09T21:43:37,509 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64. after waiting 0 ms 2024-11-09T21:43:37,509 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64. 2024-11-09T21:43:37,509 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 16065d24acc6760bfc8ee9d803a26d64 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T21:43:37,525 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnOpenScannerAsync/16065d24acc6760bfc8ee9d803a26d64/.tmp/testFamily/d989708a8a014357bc4acfa2b28b9e5e is 61, key is row-0/testFamily:testQualifier/1731188610716/Put/seqid=0 2024-11-09T21:43:37,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741866_1042 (size=5049) 2024-11-09T21:43:37,632 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1351): Waiting on 16065d24acc6760bfc8ee9d803a26d64, ab051e55337f5a7f78d0c70c373a3851 2024-11-09T21:43:37,832 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-09T21:43:37,832 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1325): Online Regions={16065d24acc6760bfc8ee9d803a26d64=testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64., ab051e55337f5a7f78d0c70c373a3851=testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851.} 2024-11-09T21:43:37,832 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1351): Waiting on 16065d24acc6760bfc8ee9d803a26d64, ab051e55337f5a7f78d0c70c373a3851 2024-11-09T21:43:37,932 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnOpenScannerAsync/16065d24acc6760bfc8ee9d803a26d64/.tmp/testFamily/d989708a8a014357bc4acfa2b28b9e5e 2024-11-09T21:43:37,946 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnOpenScannerAsync/16065d24acc6760bfc8ee9d803a26d64/.tmp/testFamily/d989708a8a014357bc4acfa2b28b9e5e as hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnOpenScannerAsync/16065d24acc6760bfc8ee9d803a26d64/testFamily/d989708a8a014357bc4acfa2b28b9e5e 2024-11-09T21:43:37,955 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnOpenScannerAsync/16065d24acc6760bfc8ee9d803a26d64/testFamily/d989708a8a014357bc4acfa2b28b9e5e, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T21:43:37,956 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 16065d24acc6760bfc8ee9d803a26d64 in 447ms, sequenceid=8, compaction requested=false 2024-11-09T21:43:37,961 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnOpenScannerAsync/16065d24acc6760bfc8ee9d803a26d64/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T21:43:37,961 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64. 2024-11-09T21:43:37,962 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 16065d24acc6760bfc8ee9d803a26d64: Waiting for close lock at 1731188617509Running coprocessor pre-close hooks at 1731188617509Disabling compacts and flushes for region at 1731188617509Disabling writes for close at 1731188617509Obtaining lock to block concurrent updates at 1731188617509Preparing flush snapshotting stores in 16065d24acc6760bfc8ee9d803a26d64 at 1731188617509Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731188617510 (+1 ms)Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64. at 1731188617510Flushing 16065d24acc6760bfc8ee9d803a26d64/testFamily: creating writer at 1731188617510Flushing 16065d24acc6760bfc8ee9d803a26d64/testFamily: appending metadata at 1731188617524 (+14 ms)Flushing 16065d24acc6760bfc8ee9d803a26d64/testFamily: closing flushed file at 1731188617524Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32fac7be: reopening flushed file at 1731188617945 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 16065d24acc6760bfc8ee9d803a26d64 in 447ms, sequenceid=8, compaction requested=false at 1731188617956 (+11 ms)Writing region close event to WAL at 1731188617957 (+1 ms)Running coprocessor post-close hooks at 1731188617961 (+4 ms)Closed at 1731188617961 2024-11-09T21:43:37,962 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1731188607656.16065d24acc6760bfc8ee9d803a26d64. 2024-11-09T21:43:37,962 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ab051e55337f5a7f78d0c70c373a3851, disabling compactions & flushes 2024-11-09T21:43:37,962 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851. 2024-11-09T21:43:37,962 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851. 2024-11-09T21:43:37,962 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851. after waiting 0 ms 2024-11-09T21:43:37,962 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851. 2024-11-09T21:43:37,962 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing ab051e55337f5a7f78d0c70c373a3851 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-09T21:43:37,979 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnOpenScanner/ab051e55337f5a7f78d0c70c373a3851/.tmp/testFamily/fae7fe8b5e384475b73b10499a06c7f5 is 61, key is row-0/testFamily:testQualifier/1731188574169/Put/seqid=0 2024-11-09T21:43:37,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741867_1043 (size=5049) 2024-11-09T21:43:38,032 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1351): Waiting on ab051e55337f5a7f78d0c70c373a3851 2024-11-09T21:43:38,233 DEBUG [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1351): Waiting on ab051e55337f5a7f78d0c70c373a3851 2024-11-09T21:43:38,386 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnOpenScanner/ab051e55337f5a7f78d0c70c373a3851/.tmp/testFamily/fae7fe8b5e384475b73b10499a06c7f5 2024-11-09T21:43:38,401 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnOpenScanner/ab051e55337f5a7f78d0c70c373a3851/.tmp/testFamily/fae7fe8b5e384475b73b10499a06c7f5 as hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnOpenScanner/ab051e55337f5a7f78d0c70c373a3851/testFamily/fae7fe8b5e384475b73b10499a06c7f5 2024-11-09T21:43:38,409 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnOpenScanner/ab051e55337f5a7f78d0c70c373a3851/testFamily/fae7fe8b5e384475b73b10499a06c7f5, entries=4, sequenceid=8, filesize=4.9 K 2024-11-09T21:43:38,410 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ab051e55337f5a7f78d0c70c373a3851 in 448ms, sequenceid=8, compaction requested=false 2024-11-09T21:43:38,415 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/data/default/testNormalScanTimeoutOnOpenScanner/ab051e55337f5a7f78d0c70c373a3851/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-09T21:43:38,416 INFO [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851. 2024-11-09T21:43:38,416 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ab051e55337f5a7f78d0c70c373a3851: Waiting for close lock at 1731188617962Running coprocessor pre-close hooks at 1731188617962Disabling compacts and flushes for region at 1731188617962Disabling writes for close at 1731188617962Obtaining lock to block concurrent updates at 1731188617962Preparing flush snapshotting stores in ab051e55337f5a7f78d0c70c373a3851 at 1731188617962Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1731188617963 (+1 ms)Flushing stores of testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851. at 1731188617963Flushing ab051e55337f5a7f78d0c70c373a3851/testFamily: creating writer at 1731188617963Flushing ab051e55337f5a7f78d0c70c373a3851/testFamily: appending metadata at 1731188617979 (+16 ms)Flushing ab051e55337f5a7f78d0c70c373a3851/testFamily: closing flushed file at 1731188617979Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@755be883: reopening flushed file at 1731188618400 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for ab051e55337f5a7f78d0c70c373a3851 in 448ms, sequenceid=8, compaction requested=false at 1731188618410 (+10 ms)Writing region close event to WAL at 1731188618411 (+1 ms)Running coprocessor post-close hooks at 1731188618416 (+5 ms)Closed at 1731188618416 2024-11-09T21:43:38,416 DEBUG [RS_CLOSE_REGION-regionserver/d86ff69d6719:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1731188571130.ab051e55337f5a7f78d0c70c373a3851. 2024-11-09T21:43:38,433 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(976): stopping server d86ff69d6719,46403,1731188512651; all regions closed. 2024-11-09T21:43:38,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741834_1010 (size=19479) 2024-11-09T21:43:38,442 DEBUG [RS:0;d86ff69d6719:46403 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/oldWALs 2024-11-09T21:43:38,442 INFO [RS:0;d86ff69d6719:46403 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d86ff69d6719%2C46403%2C1731188512651.meta:.meta(num 1731188546155) 2024-11-09T21:43:38,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741833_1009 (size=20031) 2024-11-09T21:43:38,445 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/WALs/d86ff69d6719,46403,1731188512651/d86ff69d6719%2C46403%2C1731188512651.1731188545635 not finished, retry = 0 2024-11-09T21:43:38,552 DEBUG [RS:0;d86ff69d6719:46403 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/oldWALs 2024-11-09T21:43:38,553 INFO [RS:0;d86ff69d6719:46403 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d86ff69d6719%2C46403%2C1731188512651:(num 1731188545635) 2024-11-09T21:43:38,553 DEBUG [RS:0;d86ff69d6719:46403 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:43:38,553 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T21:43:38,553 INFO [RS:0;d86ff69d6719:46403 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T21:43:38,554 INFO [RS:0;d86ff69d6719:46403 {}] hbase.ChoreService(370): Chore service for: regionserver/d86ff69d6719:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-09T21:43:38,554 INFO [RS:0;d86ff69d6719:46403 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T21:43:38,554 INFO [regionserver/d86ff69d6719:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T21:43:38,557 INFO [RS:0;d86ff69d6719:46403 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46403 2024-11-09T21:43:38,605 INFO [regionserver/d86ff69d6719:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T21:43:38,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43251-0x10122d5e4550000, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T21:43:38,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46403-0x10122d5e4550001, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d86ff69d6719,46403,1731188512651 2024-11-09T21:43:38,606 INFO [RS:0;d86ff69d6719:46403 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T21:43:38,608 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d86ff69d6719,46403,1731188512651] 2024-11-09T21:43:38,628 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d86ff69d6719,46403,1731188512651 already deleted, retry=false 2024-11-09T21:43:38,628 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d86ff69d6719,46403,1731188512651 expired; onlineServers=0 2024-11-09T21:43:38,628 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd86ff69d6719,43251,1731188511956' ***** 2024-11-09T21:43:38,628 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-09T21:43:38,629 INFO [M:0;d86ff69d6719:43251 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T21:43:38,629 INFO [M:0;d86ff69d6719:43251 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T21:43:38,629 DEBUG [M:0;d86ff69d6719:43251 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-09T21:43:38,630 DEBUG [M:0;d86ff69d6719:43251 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-09T21:43:38,630 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-09T21:43:38,630 DEBUG [master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.small.0-1731188515444 {}] cleaner.HFileCleaner(306): Exit Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.small.0-1731188515444,5,FailOnTimeoutGroup] 2024-11-09T21:43:38,630 DEBUG [master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.large.0-1731188515443 {}] cleaner.HFileCleaner(306): Exit Thread[master/d86ff69d6719:0:becomeActiveMaster-HFileCleaner.large.0-1731188515443,5,FailOnTimeoutGroup] 2024-11-09T21:43:38,630 INFO [M:0;d86ff69d6719:43251 {}] hbase.ChoreService(370): Chore service for: master/d86ff69d6719:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-09T21:43:38,631 INFO [M:0;d86ff69d6719:43251 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T21:43:38,631 DEBUG [M:0;d86ff69d6719:43251 {}] master.HMaster(1795): Stopping service threads 2024-11-09T21:43:38,631 INFO [M:0;d86ff69d6719:43251 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-09T21:43:38,631 INFO [M:0;d86ff69d6719:43251 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:43:38,633 INFO [M:0;d86ff69d6719:43251 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T21:43:38,633 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-09T21:43:38,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43251-0x10122d5e4550000, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-09T21:43:38,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43251-0x10122d5e4550000, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:43:38,639 DEBUG [M:0;d86ff69d6719:43251 {}] zookeeper.ZKUtil(347): master:43251-0x10122d5e4550000, quorum=127.0.0.1:62679, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-09T21:43:38,639 WARN [M:0;d86ff69d6719:43251 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-09T21:43:38,642 INFO [M:0;d86ff69d6719:43251 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/.lastflushedseqids 2024-11-09T21:43:38,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741868_1044 (size=649) 2024-11-09T21:43:38,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46403-0x10122d5e4550001, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T21:43:38,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46403-0x10122d5e4550001, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T21:43:38,720 INFO [RS:0;d86ff69d6719:46403 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T21:43:38,720 INFO [RS:0;d86ff69d6719:46403 {}] regionserver.HRegionServer(1031): Exiting; stopping=d86ff69d6719,46403,1731188512651; zookeeper connection closed. 2024-11-09T21:43:38,721 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1074cc05 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1074cc05 2024-11-09T21:43:38,721 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-09T21:43:39,058 INFO [M:0;d86ff69d6719:43251 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T21:43:39,059 INFO [M:0;d86ff69d6719:43251 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-09T21:43:39,059 DEBUG [M:0;d86ff69d6719:43251 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T21:43:39,059 INFO [M:0;d86ff69d6719:43251 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T21:43:39,059 DEBUG [M:0;d86ff69d6719:43251 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T21:43:39,060 DEBUG [M:0;d86ff69d6719:43251 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T21:43:39,060 DEBUG [M:0;d86ff69d6719:43251 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T21:43:39,060 INFO [M:0;d86ff69d6719:43251 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=163.01 KB heapSize=190.93 KB 2024-11-09T21:43:39,085 DEBUG [M:0;d86ff69d6719:43251 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f566a5353d2f41268a2ccee2109b4c7a is 82, key is hbase:meta,,1/info:regioninfo/1731188546232/Put/seqid=0 2024-11-09T21:43:39,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741869_1045 (size=5672) 2024-11-09T21:43:39,492 INFO [M:0;d86ff69d6719:43251 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f566a5353d2f41268a2ccee2109b4c7a 2024-11-09T21:43:39,527 DEBUG [M:0;d86ff69d6719:43251 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4de204abe63b4c39b3ca1357e01eb941 is 811, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731188547907/Put/seqid=0 2024-11-09T21:43:39,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741870_1046 (size=14072) 2024-11-09T21:43:39,828 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T21:43:39,933 INFO [M:0;d86ff69d6719:43251 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.46 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4de204abe63b4c39b3ca1357e01eb941 2024-11-09T21:43:39,944 INFO [M:0;d86ff69d6719:43251 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4de204abe63b4c39b3ca1357e01eb941 2024-11-09T21:43:39,964 DEBUG [M:0;d86ff69d6719:43251 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d92a5688a8a24fd694017ddcca7b08d6 is 69, key is d86ff69d6719,46403,1731188512651/rs:state/1731188515482/Put/seqid=0 2024-11-09T21:43:39,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741871_1047 (size=5156) 2024-11-09T21:43:40,371 INFO [M:0;d86ff69d6719:43251 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d92a5688a8a24fd694017ddcca7b08d6 2024-11-09T21:43:40,386 DEBUG [M:0;d86ff69d6719:43251 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f566a5353d2f41268a2ccee2109b4c7a as hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f566a5353d2f41268a2ccee2109b4c7a 2024-11-09T21:43:40,394 INFO [M:0;d86ff69d6719:43251 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f566a5353d2f41268a2ccee2109b4c7a, entries=8, sequenceid=317, filesize=5.5 K 2024-11-09T21:43:40,395 DEBUG [M:0;d86ff69d6719:43251 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4de204abe63b4c39b3ca1357e01eb941 as hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4de204abe63b4c39b3ca1357e01eb941 2024-11-09T21:43:40,401 INFO [M:0;d86ff69d6719:43251 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4de204abe63b4c39b3ca1357e01eb941 2024-11-09T21:43:40,401 INFO [M:0;d86ff69d6719:43251 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4de204abe63b4c39b3ca1357e01eb941, entries=33, sequenceid=317, filesize=13.7 K 2024-11-09T21:43:40,402 DEBUG [M:0;d86ff69d6719:43251 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d92a5688a8a24fd694017ddcca7b08d6 as hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d92a5688a8a24fd694017ddcca7b08d6 2024-11-09T21:43:40,408 INFO [M:0;d86ff69d6719:43251 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35923/user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d92a5688a8a24fd694017ddcca7b08d6, entries=1, sequenceid=317, filesize=5.0 K 2024-11-09T21:43:40,410 INFO [M:0;d86ff69d6719:43251 {}] regionserver.HRegion(3140): Finished flush of dataSize ~163.01 KB/166927, heapSize ~190.63 KB/195208, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1349ms, sequenceid=317, compaction requested=false 2024-11-09T21:43:40,411 INFO [M:0;d86ff69d6719:43251 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T21:43:40,411 DEBUG [M:0;d86ff69d6719:43251 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731188619059Disabling compacts and flushes for region at 1731188619059Disabling writes for close at 1731188619060 (+1 ms)Obtaining lock to block concurrent updates at 1731188619060Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731188619060Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166927, getHeapSize=195448, getOffHeapSize=0, getCellsCount=364 at 1731188619061 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731188619062 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731188619062Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731188619084 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731188619085 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731188619507 (+422 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731188619527 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731188619527Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731188619945 (+418 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731188619963 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731188619963Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10265c55: reopening flushed file at 1731188620385 (+422 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38ae0ef8: reopening flushed file at 1731188620394 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24d23184: reopening flushed file at 1731188620401 (+7 ms)Finished flush of dataSize ~163.01 KB/166927, heapSize ~190.63 KB/195208, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1349ms, sequenceid=317, compaction requested=false at 1731188620410 (+9 ms)Writing region close event to WAL at 1731188620411 (+1 ms)Closed at 1731188620411 2024-11-09T21:43:40,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42537 is added to blk_1073741830_1006 (size=188139) 2024-11-09T21:43:40,414 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/6f4a9063-6cb5-edd9-3dcd-66e39b059118/MasterData/WALs/d86ff69d6719,43251,1731188511956/d86ff69d6719%2C43251%2C1731188511956.1731188514786 not finished, retry = 0 2024-11-09T21:43:40,516 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T21:43:40,516 INFO [M:0;d86ff69d6719:43251 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-09T21:43:40,517 INFO [M:0;d86ff69d6719:43251 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43251 2024-11-09T21:43:40,518 INFO [M:0;d86ff69d6719:43251 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T21:43:40,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43251-0x10122d5e4550000, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T21:43:40,681 INFO [M:0;d86ff69d6719:43251 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T21:43:40,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43251-0x10122d5e4550000, quorum=127.0.0.1:62679, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T21:43:40,687 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@429b6cf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T21:43:40,690 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@406d11bc{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-09T21:43:40,690 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-09T21:43:40,690 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7f8f7398{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-09T21:43:40,690 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@79afe1c5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/hadoop.log.dir/,STOPPED} 2024-11-09T21:43:40,695 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-09T21:43:40,695 WARN [BP-1022936143-172.17.0.2-1731188508680 heartbeating to localhost/127.0.0.1:35923 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-09T21:43:40,695 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-09T21:43:40,695 WARN [BP-1022936143-172.17.0.2-1731188508680 heartbeating to localhost/127.0.0.1:35923 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1022936143-172.17.0.2-1731188508680 (Datanode Uuid 08eec006-7af3-48aa-8495-4ce0dc4f9008) service to localhost/127.0.0.1:35923 2024-11-09T21:43:40,696 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/cluster_7bb766c4-c195-17b1-f40b-f54a9445f0b2/data/data1/current/BP-1022936143-172.17.0.2-1731188508680 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T21:43:40,696 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/cluster_7bb766c4-c195-17b1-f40b-f54a9445f0b2/data/data2/current/BP-1022936143-172.17.0.2-1731188508680 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T21:43:40,697 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-09T21:43:40,707 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@40b93e99{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-09T21:43:40,708 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4e84709f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-09T21:43:40,708 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-09T21:43:40,708 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@38f794dd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-09T21:43:40,708 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@13a4dda0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8efdc891-a4ca-d1e3-f76a-60edd211a166/hadoop.log.dir/,STOPPED} 2024-11-09T21:43:40,717 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-09T21:43:40,739 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down