2024-11-27 17:36:48,324 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-27 17:36:48,340 main DEBUG Took 0.014040 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-27 17:36:48,340 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-27 17:36:48,341 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-27 17:36:48,342 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-27 17:36:48,343 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 17:36:48,352 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-27 17:36:48,364 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 17:36:48,365 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 17:36:48,366 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 17:36:48,367 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 17:36:48,368 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 17:36:48,368 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 17:36:48,370 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 17:36:48,370 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 17:36:48,371 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 17:36:48,372 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 17:36:48,373 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 17:36:48,373 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 17:36:48,374 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 17:36:48,375 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 17:36:48,375 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 17:36:48,376 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 17:36:48,377 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 17:36:48,377 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 17:36:48,378 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 17:36:48,378 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 17:36:48,379 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 17:36:48,379 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 17:36:48,380 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 17:36:48,380 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 17:36:48,381 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 17:36:48,382 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-27 17:36:48,384 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 17:36:48,385 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-27 17:36:48,388 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-27 17:36:48,388 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-27 17:36:48,390 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-27 17:36:48,391 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-27 17:36:48,402 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-27 17:36:48,405 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-27 17:36:48,408 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-27 17:36:48,408 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-27 17:36:48,409 main DEBUG createAppenders(={Console}) 2024-11-27 17:36:48,410 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 initialized 2024-11-27 17:36:48,410 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 2024-11-27 17:36:48,411 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5fb759d6 OK. 2024-11-27 17:36:48,411 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-27 17:36:48,412 main DEBUG OutputStream closed 2024-11-27 17:36:48,412 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-27 17:36:48,413 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-27 17:36:48,413 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5b03b9fe OK 2024-11-27 17:36:48,496 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-27 17:36:48,498 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-27 17:36:48,499 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-27 17:36:48,500 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-27 17:36:48,501 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-27 17:36:48,501 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-27 17:36:48,502 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-27 17:36:48,502 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-27 17:36:48,502 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-27 17:36:48,503 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-27 17:36:48,503 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-27 17:36:48,503 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-27 17:36:48,504 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-27 17:36:48,504 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-27 17:36:48,504 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-27 17:36:48,505 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-27 17:36:48,505 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-27 17:36:48,506 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-27 17:36:48,508 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27 17:36:48,508 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5bf8fa12) with optional ClassLoader: null 2024-11-27 17:36:48,508 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-27 17:36:48,509 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@5bf8fa12] started OK. 2024-11-27T17:36:48,522 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestClientScannerTimeouts timeout: 13 mins 2024-11-27 17:36:48,524 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-27 17:36:48,524 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27T17:36:48,754 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c 2024-11-27T17:36:48,772 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-27T17:36:48,787 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/cluster_6034e5b1-ec67-3510-6ba0-128f6811cfb0, deleteOnExit=true 2024-11-27T17:36:48,787 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-27T17:36:48,788 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/test.cache.data in system properties and HBase conf 2024-11-27T17:36:48,789 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/hadoop.tmp.dir in system properties and HBase conf 2024-11-27T17:36:48,789 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/hadoop.log.dir in system properties and HBase conf 2024-11-27T17:36:48,790 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-27T17:36:48,791 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-27T17:36:48,791 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-27T17:36:48,865 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-27T17:36:48,948 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T17:36:48,953 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-27T17:36:48,954 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-27T17:36:48,955 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-27T17:36:48,956 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T17:36:48,956 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-27T17:36:48,957 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-27T17:36:48,957 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T17:36:48,958 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T17:36:48,959 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-27T17:36:48,959 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/nfs.dump.dir in system properties and HBase conf 2024-11-27T17:36:48,960 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/java.io.tmpdir in system properties and HBase conf 2024-11-27T17:36:48,961 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T17:36:48,961 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-27T17:36:48,962 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-27T17:36:49,991 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-27T17:36:50,059 INFO [Time-limited test {}] log.Log(170): Logging initialized @2391ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-27T17:36:50,124 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T17:36:50,182 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T17:36:50,203 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T17:36:50,203 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T17:36:50,204 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-27T17:36:50,215 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T17:36:50,218 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/hadoop.log.dir/,AVAILABLE} 2024-11-27T17:36:50,219 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T17:36:50,390 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@eb30bc1{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/java.io.tmpdir/jetty-localhost-40019-hadoop-hdfs-3_4_1-tests_jar-_-any-12374936579031617676/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T17:36:50,400 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:40019} 2024-11-27T17:36:50,400 INFO [Time-limited test {}] server.Server(415): Started @2733ms 2024-11-27T17:36:51,019 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T17:36:51,026 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T17:36:51,027 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T17:36:51,028 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T17:36:51,028 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-27T17:36:51,029 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/hadoop.log.dir/,AVAILABLE} 2024-11-27T17:36:51,029 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T17:36:51,127 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5976fb41{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/java.io.tmpdir/jetty-localhost-46269-hadoop-hdfs-3_4_1-tests_jar-_-any-17833214714687709286/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T17:36:51,128 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:46269} 2024-11-27T17:36:51,128 INFO [Time-limited test {}] server.Server(415): Started @3461ms 2024-11-27T17:36:51,182 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-27T17:36:52,357 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/cluster_6034e5b1-ec67-3510-6ba0-128f6811cfb0/data/data2/current/BP-358449881-172.17.0.3-1732729009514/current, will proceed with Du for space computation calculation, 2024-11-27T17:36:52,357 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/cluster_6034e5b1-ec67-3510-6ba0-128f6811cfb0/data/data1/current/BP-358449881-172.17.0.3-1732729009514/current, will proceed with Du for space computation calculation, 2024-11-27T17:36:52,384 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-27T17:36:52,429 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe1d436ba96176aa5 with lease ID 0x80eca627ec180214: Processing first storage report for DS-13682c8a-7be2-428c-804d-def2367ffa31 from datanode DatanodeRegistration(127.0.0.1:32923, datanodeUuid=4bfa520b-375c-403e-933e-34e2b075d46f, infoPort=34727, infoSecurePort=0, ipcPort=35627, storageInfo=lv=-57;cid=testClusterID;nsid=1117167775;c=1732729009514) 2024-11-27T17:36:52,430 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe1d436ba96176aa5 with lease ID 0x80eca627ec180214: from storage DS-13682c8a-7be2-428c-804d-def2367ffa31 node DatanodeRegistration(127.0.0.1:32923, datanodeUuid=4bfa520b-375c-403e-933e-34e2b075d46f, infoPort=34727, infoSecurePort=0, ipcPort=35627, storageInfo=lv=-57;cid=testClusterID;nsid=1117167775;c=1732729009514), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-27T17:36:52,430 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe1d436ba96176aa5 with lease ID 0x80eca627ec180214: Processing first storage report for DS-2303a56c-f17c-4180-af7e-9b721ed230cc from datanode DatanodeRegistration(127.0.0.1:32923, datanodeUuid=4bfa520b-375c-403e-933e-34e2b075d46f, infoPort=34727, infoSecurePort=0, ipcPort=35627, storageInfo=lv=-57;cid=testClusterID;nsid=1117167775;c=1732729009514) 2024-11-27T17:36:52,431 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe1d436ba96176aa5 with lease ID 0x80eca627ec180214: from storage DS-2303a56c-f17c-4180-af7e-9b721ed230cc node DatanodeRegistration(127.0.0.1:32923, datanodeUuid=4bfa520b-375c-403e-933e-34e2b075d46f, infoPort=34727, infoSecurePort=0, ipcPort=35627, storageInfo=lv=-57;cid=testClusterID;nsid=1117167775;c=1732729009514), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-27T17:36:52,474 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c 2024-11-27T17:36:52,535 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/cluster_6034e5b1-ec67-3510-6ba0-128f6811cfb0/zookeeper_0, clientPort=64480, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/cluster_6034e5b1-ec67-3510-6ba0-128f6811cfb0/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/cluster_6034e5b1-ec67-3510-6ba0-128f6811cfb0/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-27T17:36:52,544 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64480 2024-11-27T17:36:52,553 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T17:36:52,555 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T17:36:52,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741825_1001 (size=7) 2024-11-27T17:36:53,158 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f with version=8 2024-11-27T17:36:53,158 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/hbase-staging 2024-11-27T17:36:53,255 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-27T17:36:53,466 INFO [Time-limited test {}] client.ConnectionUtils(128): master/8f0673442175:0 server-side Connection retries=9 2024-11-27T17:36:53,474 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T17:36:53,475 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T17:36:53,479 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T17:36:53,479 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T17:36:53,479 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T17:36:53,622 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-27T17:36:53,679 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-27T17:36:53,689 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-27T17:36:53,692 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T17:36:53,714 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 113998 (auto-detected) 2024-11-27T17:36:53,716 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-27T17:36:53,734 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46243 2024-11-27T17:36:53,761 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46243 connecting to ZooKeeper ensemble=127.0.0.1:64480 2024-11-27T17:36:53,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:462430x0, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T17:36:53,850 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46243-0x1017d5e7cd30000 connected 2024-11-27T17:36:54,178 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T17:36:54,181 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T17:36:54,194 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46243-0x1017d5e7cd30000, quorum=127.0.0.1:64480, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T17:36:54,199 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f, hbase.cluster.distributed=false 2024-11-27T17:36:54,221 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46243-0x1017d5e7cd30000, quorum=127.0.0.1:64480, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T17:36:54,226 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46243 2024-11-27T17:36:54,227 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46243 2024-11-27T17:36:54,227 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46243 2024-11-27T17:36:54,228 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46243 2024-11-27T17:36:54,228 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46243 2024-11-27T17:36:54,327 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/8f0673442175:0 server-side Connection retries=9 2024-11-27T17:36:54,329 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T17:36:54,329 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T17:36:54,329 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T17:36:54,330 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T17:36:54,330 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T17:36:54,333 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T17:36:54,336 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T17:36:54,337 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43899 2024-11-27T17:36:54,339 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43899 connecting to ZooKeeper ensemble=127.0.0.1:64480 2024-11-27T17:36:54,340 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T17:36:54,343 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T17:36:54,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:438990x0, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T17:36:54,366 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:438990x0, quorum=127.0.0.1:64480, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T17:36:54,366 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43899-0x1017d5e7cd30001 connected 2024-11-27T17:36:54,369 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T17:36:54,376 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T17:36:54,379 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43899-0x1017d5e7cd30001, quorum=127.0.0.1:64480, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T17:36:54,383 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43899-0x1017d5e7cd30001, quorum=127.0.0.1:64480, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T17:36:54,384 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43899 2024-11-27T17:36:54,384 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43899 2024-11-27T17:36:54,385 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43899 2024-11-27T17:36:54,388 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43899 2024-11-27T17:36:54,388 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43899 2024-11-27T17:36:54,404 DEBUG [M:0;8f0673442175:46243 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;8f0673442175:46243 2024-11-27T17:36:54,405 INFO [master/8f0673442175:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/8f0673442175,46243,1732729013300 2024-11-27T17:36:54,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43899-0x1017d5e7cd30001, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T17:36:54,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46243-0x1017d5e7cd30000, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T17:36:54,472 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46243-0x1017d5e7cd30000, quorum=127.0.0.1:64480, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/8f0673442175,46243,1732729013300 2024-11-27T17:36:54,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43899-0x1017d5e7cd30001, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T17:36:54,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46243-0x1017d5e7cd30000, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T17:36:54,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43899-0x1017d5e7cd30001, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T17:36:54,558 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46243-0x1017d5e7cd30000, quorum=127.0.0.1:64480, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-27T17:36:54,560 INFO [master/8f0673442175:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/8f0673442175,46243,1732729013300 from backup master directory 2024-11-27T17:36:54,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46243-0x1017d5e7cd30000, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/8f0673442175,46243,1732729013300 2024-11-27T17:36:54,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43899-0x1017d5e7cd30001, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T17:36:54,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46243-0x1017d5e7cd30000, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T17:36:54,609 WARN [master/8f0673442175:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T17:36:54,610 INFO [master/8f0673442175:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=8f0673442175,46243,1732729013300 2024-11-27T17:36:54,614 INFO [master/8f0673442175:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-27T17:36:54,616 INFO [master/8f0673442175:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-27T17:36:54,672 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/hbase.id] with ID: 1784d3b5-6fc6-48b8-ae1a-00e4efc5cff5 2024-11-27T17:36:54,673 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/.tmp/hbase.id 2024-11-27T17:36:54,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741826_1002 (size=42) 2024-11-27T17:36:55,086 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/.tmp/hbase.id]:[hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/hbase.id] 2024-11-27T17:36:55,145 INFO [master/8f0673442175:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T17:36:55,150 INFO [master/8f0673442175:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-27T17:36:55,169 INFO [master/8f0673442175:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-27T17:36:55,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43899-0x1017d5e7cd30001, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T17:36:55,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46243-0x1017d5e7cd30000, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T17:36:55,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741827_1003 (size=196) 2024-11-27T17:36:55,657 INFO [master/8f0673442175:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T17:36:55,661 INFO [master/8f0673442175:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-27T17:36:55,681 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T17:36:55,687 INFO [master/8f0673442175:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T17:36:55,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741828_1004 (size=1189) 2024-11-27T17:36:56,142 INFO [master/8f0673442175:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/MasterData/data/master/store 2024-11-27T17:36:56,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741829_1005 (size=34) 2024-11-27T17:36:56,570 INFO [master/8f0673442175:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-27T17:36:56,574 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T17:36:56,575 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T17:36:56,575 INFO [master/8f0673442175:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T17:36:56,575 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T17:36:56,577 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T17:36:56,577 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T17:36:56,577 INFO [master/8f0673442175:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T17:36:56,578 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732729016575Disabling compacts and flushes for region at 1732729016575Disabling writes for close at 1732729016577 (+2 ms)Writing region close event to WAL at 1732729016577Closed at 1732729016577 2024-11-27T17:36:56,580 WARN [master/8f0673442175:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/MasterData/data/master/store/.initializing 2024-11-27T17:36:56,580 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/MasterData/WALs/8f0673442175,46243,1732729013300 2024-11-27T17:36:56,589 INFO [master/8f0673442175:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T17:36:56,603 INFO [master/8f0673442175:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8f0673442175%2C46243%2C1732729013300, suffix=, logDir=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/MasterData/WALs/8f0673442175,46243,1732729013300, archiveDir=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/MasterData/oldWALs, maxLogs=10 2024-11-27T17:36:56,625 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/MasterData/WALs/8f0673442175,46243,1732729013300/8f0673442175%2C46243%2C1732729013300.1732729016607, exclude list is [], retry=0 2024-11-27T17:36:56,643 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32923,DS-13682c8a-7be2-428c-804d-def2367ffa31,DISK] 2024-11-27T17:36:56,645 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-27T17:36:56,678 INFO [master/8f0673442175:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/MasterData/WALs/8f0673442175,46243,1732729013300/8f0673442175%2C46243%2C1732729013300.1732729016607 2024-11-27T17:36:56,680 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34727:34727)] 2024-11-27T17:36:56,680 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-27T17:36:56,680 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T17:36:56,683 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T17:36:56,685 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T17:36:56,721 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T17:36:56,746 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-27T17:36:56,749 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:36:56,751 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T17:36:56,752 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T17:36:56,755 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-27T17:36:56,756 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:36:56,757 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T17:36:56,757 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T17:36:56,760 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-27T17:36:56,761 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:36:56,762 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T17:36:56,762 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T17:36:56,765 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-27T17:36:56,765 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:36:56,766 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T17:36:56,767 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T17:36:56,770 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T17:36:56,772 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T17:36:56,778 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T17:36:56,779 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T17:36:56,784 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T17:36:56,788 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T17:36:56,793 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T17:36:56,794 INFO [master/8f0673442175:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73487802, jitterRate=0.09505358338356018}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T17:36:56,803 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732729016697Initializing all the Stores at 1732729016700 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732729016700Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732729016701 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732729016701Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732729016701Cleaning up temporary data from old regions at 1732729016779 (+78 ms)Region opened successfully at 1732729016802 (+23 ms) 2024-11-27T17:36:56,804 INFO [master/8f0673442175:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-27T17:36:56,837 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b41aa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8f0673442175/172.17.0.3:0 2024-11-27T17:36:56,872 INFO [master/8f0673442175:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-27T17:36:56,886 INFO [master/8f0673442175:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-27T17:36:56,886 INFO [master/8f0673442175:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-27T17:36:56,889 INFO [master/8f0673442175:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T17:36:56,890 INFO [master/8f0673442175:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-27T17:36:56,894 INFO [master/8f0673442175:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-27T17:36:56,894 INFO [master/8f0673442175:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-27T17:36:56,918 INFO [master/8f0673442175:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-27T17:36:56,927 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46243-0x1017d5e7cd30000, quorum=127.0.0.1:64480, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-27T17:36:56,950 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-27T17:36:56,954 INFO [master/8f0673442175:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-27T17:36:56,955 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46243-0x1017d5e7cd30000, quorum=127.0.0.1:64480, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-27T17:36:56,964 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-27T17:36:56,967 INFO [master/8f0673442175:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-27T17:36:56,972 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46243-0x1017d5e7cd30000, quorum=127.0.0.1:64480, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-27T17:36:56,982 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-27T17:36:56,984 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46243-0x1017d5e7cd30000, quorum=127.0.0.1:64480, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-27T17:36:56,996 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-27T17:36:57,022 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46243-0x1017d5e7cd30000, quorum=127.0.0.1:64480, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-27T17:36:57,035 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-27T17:36:57,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46243-0x1017d5e7cd30000, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T17:36:57,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43899-0x1017d5e7cd30001, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T17:36:57,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46243-0x1017d5e7cd30000, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T17:36:57,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43899-0x1017d5e7cd30001, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T17:36:57,051 INFO [master/8f0673442175:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=8f0673442175,46243,1732729013300, sessionid=0x1017d5e7cd30000, setting cluster-up flag (Was=false) 2024-11-27T17:36:57,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46243-0x1017d5e7cd30000, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T17:36:57,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43899-0x1017d5e7cd30001, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T17:36:57,111 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-27T17:36:57,113 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8f0673442175,46243,1732729013300 2024-11-27T17:36:57,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46243-0x1017d5e7cd30000, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T17:36:57,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43899-0x1017d5e7cd30001, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T17:36:57,164 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-27T17:36:57,167 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8f0673442175,46243,1732729013300 2024-11-27T17:36:57,174 INFO [master/8f0673442175:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-27T17:36:57,196 INFO [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(746): ClusterId : 1784d3b5-6fc6-48b8-ae1a-00e4efc5cff5 2024-11-27T17:36:57,198 DEBUG [RS:0;8f0673442175:43899 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T17:36:57,209 DEBUG [RS:0;8f0673442175:43899 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T17:36:57,209 DEBUG [RS:0;8f0673442175:43899 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T17:36:57,219 DEBUG [RS:0;8f0673442175:43899 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T17:36:57,220 DEBUG [RS:0;8f0673442175:43899 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@555d5a6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8f0673442175/172.17.0.3:0 2024-11-27T17:36:57,235 DEBUG [RS:0;8f0673442175:43899 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;8f0673442175:43899 2024-11-27T17:36:57,237 INFO [RS:0;8f0673442175:43899 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T17:36:57,238 INFO [RS:0;8f0673442175:43899 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T17:36:57,238 DEBUG [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T17:36:57,240 INFO [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(2659): reportForDuty to master=8f0673442175,46243,1732729013300 with port=43899, startcode=1732729014292 2024-11-27T17:36:57,251 DEBUG [RS:0;8f0673442175:43899 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T17:36:57,250 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T17:36:57,260 INFO [master/8f0673442175:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-27T17:36:57,268 INFO [master/8f0673442175:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-27T17:36:57,275 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 8f0673442175,46243,1732729013300 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-27T17:36:57,284 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38433, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T17:36:57,285 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/8f0673442175:0, corePoolSize=5, maxPoolSize=5 2024-11-27T17:36:57,285 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/8f0673442175:0, corePoolSize=5, maxPoolSize=5 2024-11-27T17:36:57,285 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/8f0673442175:0, corePoolSize=5, maxPoolSize=5 2024-11-27T17:36:57,285 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/8f0673442175:0, corePoolSize=5, maxPoolSize=5 2024-11-27T17:36:57,286 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/8f0673442175:0, corePoolSize=10, maxPoolSize=10 2024-11-27T17:36:57,286 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/8f0673442175:0, corePoolSize=1, maxPoolSize=1 2024-11-27T17:36:57,286 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/8f0673442175:0, corePoolSize=2, maxPoolSize=2 2024-11-27T17:36:57,286 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/8f0673442175:0, corePoolSize=1, maxPoolSize=1 2024-11-27T17:36:57,289 INFO [master/8f0673442175:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732729047289 2024-11-27T17:36:57,291 INFO [master/8f0673442175:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-27T17:36:57,292 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T17:36:57,292 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46243 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-27T17:36:57,292 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-27T17:36:57,292 INFO [master/8f0673442175:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-27T17:36:57,297 INFO [master/8f0673442175:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-27T17:36:57,297 INFO [master/8f0673442175:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-27T17:36:57,298 INFO [master/8f0673442175:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-27T17:36:57,298 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:36:57,298 INFO [master/8f0673442175:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-27T17:36:57,298 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-27T17:36:57,299 INFO [master/8f0673442175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T17:36:57,304 INFO [master/8f0673442175:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-27T17:36:57,306 INFO [master/8f0673442175:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-27T17:36:57,306 INFO [master/8f0673442175:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-27T17:36:57,309 INFO [master/8f0673442175:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-27T17:36:57,309 INFO [master/8f0673442175:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-27T17:36:57,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741831_1007 (size=1321) 2024-11-27T17:36:57,314 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-27T17:36:57,314 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f 2024-11-27T17:36:57,320 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/8f0673442175:0:becomeActiveMaster-HFileCleaner.large.0-1732729017311,5,FailOnTimeoutGroup] 2024-11-27T17:36:57,323 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/8f0673442175:0:becomeActiveMaster-HFileCleaner.small.0-1732729017320,5,FailOnTimeoutGroup] 2024-11-27T17:36:57,323 INFO [master/8f0673442175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T17:36:57,323 INFO [master/8f0673442175:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-27T17:36:57,325 INFO [master/8f0673442175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-27T17:36:57,325 INFO [master/8f0673442175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-27T17:36:57,393 DEBUG [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-27T17:36:57,393 WARN [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 30000 ms and then retrying. 2024-11-27T17:36:57,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741832_1008 (size=32) 2024-11-27T17:36:57,799 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T17:36:57,802 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T17:36:57,804 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T17:36:57,805 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:36:57,806 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T17:36:57,806 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T17:36:57,808 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T17:36:57,808 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:36:57,809 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T17:36:57,809 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T17:36:57,812 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T17:36:57,812 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:36:57,813 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T17:36:57,813 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T17:36:57,815 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T17:36:57,815 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:36:57,816 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T17:36:57,816 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T17:36:57,818 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/meta/1588230740 2024-11-27T17:36:57,819 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/meta/1588230740 2024-11-27T17:36:57,822 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T17:36:57,822 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T17:36:57,823 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T17:36:57,826 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T17:36:57,830 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T17:36:57,831 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64892596, jitterRate=-0.03302496671676636}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T17:36:57,837 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732729017799Initializing all the Stores at 1732729017801 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732729017801Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732729017802 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732729017802Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732729017802Cleaning up temporary data from old regions at 1732729017822 (+20 ms)Region opened successfully at 1732729017837 (+15 ms) 2024-11-27T17:36:57,837 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T17:36:57,837 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T17:36:57,838 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T17:36:57,838 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T17:36:57,838 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T17:36:57,843 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T17:36:57,843 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732729017837Disabling compacts and flushes for region at 1732729017837Disabling writes for close at 1732729017838 (+1 ms)Writing region close event to WAL at 1732729017842 (+4 ms)Closed at 1732729017843 (+1 ms) 2024-11-27T17:36:57,847 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T17:36:57,847 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-27T17:36:57,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-27T17:36:57,861 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T17:36:57,864 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-27T17:36:58,015 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:36:59,016 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:00,017 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:01,018 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:02,018 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:03,019 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:03,676 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-27T17:37:03,677 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-27T17:37:04,020 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:05,021 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:06,022 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:07,022 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:08,023 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:09,024 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:10,025 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:11,026 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:12,027 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:13,027 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:14,028 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:15,029 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:16,030 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:17,031 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:18,032 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:19,034 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:20,035 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:21,036 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:22,037 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:22,472 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T17:37:23,038 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:24,039 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:25,040 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:26,041 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:27,042 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:27,396 INFO [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(2659): reportForDuty to master=8f0673442175,46243,1732729013300 with port=43899, startcode=1732729014292 2024-11-27T17:37:27,400 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46243 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 8f0673442175,43899,1732729014292 2024-11-27T17:37:27,404 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46243 {}] master.ServerManager(517): Registering regionserver=8f0673442175,43899,1732729014292 2024-11-27T17:37:27,412 DEBUG [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f 2024-11-27T17:37:27,412 DEBUG [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40815 2024-11-27T17:37:27,412 DEBUG [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T17:37:27,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46243-0x1017d5e7cd30000, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T17:37:27,488 DEBUG [RS:0;8f0673442175:43899 {}] zookeeper.ZKUtil(111): regionserver:43899-0x1017d5e7cd30001, quorum=127.0.0.1:64480, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/8f0673442175,43899,1732729014292 2024-11-27T17:37:27,489 WARN [RS:0;8f0673442175:43899 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T17:37:27,489 INFO [RS:0;8f0673442175:43899 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T17:37:27,490 DEBUG [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/WALs/8f0673442175,43899,1732729014292 2024-11-27T17:37:27,494 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [8f0673442175,43899,1732729014292] 2024-11-27T17:37:27,515 INFO [RS:0;8f0673442175:43899 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T17:37:27,526 INFO [RS:0;8f0673442175:43899 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T17:37:27,531 INFO [RS:0;8f0673442175:43899 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T17:37:27,531 INFO [RS:0;8f0673442175:43899 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T17:37:27,532 INFO [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T17:37:27,537 INFO [RS:0;8f0673442175:43899 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T17:37:27,538 INFO [RS:0;8f0673442175:43899 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T17:37:27,538 DEBUG [RS:0;8f0673442175:43899 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/8f0673442175:0, corePoolSize=1, maxPoolSize=1 2024-11-27T17:37:27,538 DEBUG [RS:0;8f0673442175:43899 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/8f0673442175:0, corePoolSize=1, maxPoolSize=1 2024-11-27T17:37:27,539 DEBUG [RS:0;8f0673442175:43899 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0, corePoolSize=1, maxPoolSize=1 2024-11-27T17:37:27,539 DEBUG [RS:0;8f0673442175:43899 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/8f0673442175:0, corePoolSize=1, maxPoolSize=1 2024-11-27T17:37:27,539 DEBUG [RS:0;8f0673442175:43899 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/8f0673442175:0, corePoolSize=1, maxPoolSize=1 2024-11-27T17:37:27,539 DEBUG [RS:0;8f0673442175:43899 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/8f0673442175:0, corePoolSize=2, maxPoolSize=2 2024-11-27T17:37:27,539 DEBUG [RS:0;8f0673442175:43899 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/8f0673442175:0, corePoolSize=1, maxPoolSize=1 2024-11-27T17:37:27,539 DEBUG [RS:0;8f0673442175:43899 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/8f0673442175:0, corePoolSize=1, maxPoolSize=1 2024-11-27T17:37:27,539 DEBUG [RS:0;8f0673442175:43899 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/8f0673442175:0, corePoolSize=1, maxPoolSize=1 2024-11-27T17:37:27,540 DEBUG [RS:0;8f0673442175:43899 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/8f0673442175:0, corePoolSize=1, maxPoolSize=1 2024-11-27T17:37:27,540 DEBUG [RS:0;8f0673442175:43899 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/8f0673442175:0, corePoolSize=1, maxPoolSize=1 2024-11-27T17:37:27,540 DEBUG [RS:0;8f0673442175:43899 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/8f0673442175:0, corePoolSize=1, maxPoolSize=1 2024-11-27T17:37:27,540 DEBUG [RS:0;8f0673442175:43899 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/8f0673442175:0, corePoolSize=3, maxPoolSize=3 2024-11-27T17:37:27,540 DEBUG [RS:0;8f0673442175:43899 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/8f0673442175:0, corePoolSize=3, maxPoolSize=3 2024-11-27T17:37:27,541 INFO [RS:0;8f0673442175:43899 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T17:37:27,541 INFO [RS:0;8f0673442175:43899 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T17:37:27,541 INFO [RS:0;8f0673442175:43899 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T17:37:27,541 INFO [RS:0;8f0673442175:43899 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T17:37:27,541 INFO [RS:0;8f0673442175:43899 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T17:37:27,541 INFO [RS:0;8f0673442175:43899 {}] hbase.ChoreService(168): Chore ScheduledChore name=8f0673442175,43899,1732729014292-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T17:37:27,558 INFO [RS:0;8f0673442175:43899 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T17:37:27,560 INFO [RS:0;8f0673442175:43899 {}] hbase.ChoreService(168): Chore ScheduledChore name=8f0673442175,43899,1732729014292-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T17:37:27,560 INFO [RS:0;8f0673442175:43899 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T17:37:27,561 INFO [RS:0;8f0673442175:43899 {}] regionserver.Replication(171): 8f0673442175,43899,1732729014292 started 2024-11-27T17:37:27,576 INFO [RS:0;8f0673442175:43899 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T17:37:27,577 INFO [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1482): Serving as 8f0673442175,43899,1732729014292, RpcServer on 8f0673442175/172.17.0.3:43899, sessionid=0x1017d5e7cd30001 2024-11-27T17:37:27,577 DEBUG [RS:0;8f0673442175:43899 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T17:37:27,578 DEBUG [RS:0;8f0673442175:43899 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 8f0673442175,43899,1732729014292 2024-11-27T17:37:27,578 DEBUG [RS:0;8f0673442175:43899 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8f0673442175,43899,1732729014292' 2024-11-27T17:37:27,578 DEBUG [RS:0;8f0673442175:43899 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T17:37:27,579 DEBUG [RS:0;8f0673442175:43899 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T17:37:27,580 DEBUG [RS:0;8f0673442175:43899 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T17:37:27,580 DEBUG [RS:0;8f0673442175:43899 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T17:37:27,580 DEBUG [RS:0;8f0673442175:43899 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 8f0673442175,43899,1732729014292 2024-11-27T17:37:27,580 DEBUG [RS:0;8f0673442175:43899 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8f0673442175,43899,1732729014292' 2024-11-27T17:37:27,580 DEBUG [RS:0;8f0673442175:43899 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T17:37:27,581 DEBUG [RS:0;8f0673442175:43899 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T17:37:27,582 DEBUG [RS:0;8f0673442175:43899 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T17:37:27,582 INFO [RS:0;8f0673442175:43899 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T17:37:27,582 INFO [RS:0;8f0673442175:43899 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T17:37:28,043 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:29,044 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:30,045 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:31,046 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:32,047 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:33,048 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:34,049 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:35,050 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:36,051 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:37,052 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:38,053 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:39,054 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:40,055 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:41,056 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:42,058 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:43,059 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:44,060 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:45,061 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:46,062 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:47,064 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:48,065 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:49,066 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:50,067 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:51,068 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:52,069 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:52,472 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T17:37:53,070 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:54,071 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:55,072 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:56,073 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:57,074 WARN [8f0673442175:46243 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T17:37:57,597 INFO [RS:0;8f0673442175:43899 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T17:37:57,600 INFO [RS:0;8f0673442175:43899 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8f0673442175%2C43899%2C1732729014292, suffix=, logDir=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/WALs/8f0673442175,43899,1732729014292, archiveDir=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/oldWALs, maxLogs=32 2024-11-27T17:37:57,614 DEBUG [RS:0;8f0673442175:43899 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/WALs/8f0673442175,43899,1732729014292/8f0673442175%2C43899%2C1732729014292.1732729077602, exclude list is [], retry=0 2024-11-27T17:37:57,619 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32923,DS-13682c8a-7be2-428c-804d-def2367ffa31,DISK] 2024-11-27T17:37:57,622 INFO [RS:0;8f0673442175:43899 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/WALs/8f0673442175,43899,1732729014292/8f0673442175%2C43899%2C1732729014292.1732729077602 2024-11-27T17:37:57,622 DEBUG [RS:0;8f0673442175:43899 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34727:34727)] 2024-11-27T17:37:57,829 DEBUG [8f0673442175:46243 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-27T17:37:57,844 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=8f0673442175,43899,1732729014292 2024-11-27T17:37:57,850 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8f0673442175,43899,1732729014292, state=OPENING 2024-11-27T17:37:57,928 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-27T17:37:57,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46243-0x1017d5e7cd30000, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T17:37:57,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43899-0x1017d5e7cd30001, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T17:37:57,941 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T17:37:57,941 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T17:37:57,944 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T17:37:57,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=8f0673442175,43899,1732729014292}] 2024-11-27T17:37:58,129 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T17:37:58,132 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57383, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T17:37:58,142 INFO [RS_OPEN_META-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-27T17:37:58,142 INFO [RS_OPEN_META-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T17:37:58,142 INFO [RS_OPEN_META-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-27T17:37:58,146 INFO [RS_OPEN_META-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8f0673442175%2C43899%2C1732729014292.meta, suffix=.meta, logDir=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/WALs/8f0673442175,43899,1732729014292, archiveDir=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/oldWALs, maxLogs=32 2024-11-27T17:37:58,163 DEBUG [RS_OPEN_META-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/WALs/8f0673442175,43899,1732729014292/8f0673442175%2C43899%2C1732729014292.meta.1732729078148.meta, exclude list is [], retry=0 2024-11-27T17:37:58,167 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:32923,DS-13682c8a-7be2-428c-804d-def2367ffa31,DISK] 2024-11-27T17:37:58,170 INFO [RS_OPEN_META-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/WALs/8f0673442175,43899,1732729014292/8f0673442175%2C43899%2C1732729014292.meta.1732729078148.meta 2024-11-27T17:37:58,170 DEBUG [RS_OPEN_META-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34727:34727)] 2024-11-27T17:37:58,171 DEBUG [RS_OPEN_META-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-27T17:37:58,172 DEBUG [RS_OPEN_META-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-27T17:37:58,175 DEBUG [RS_OPEN_META-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-27T17:37:58,179 INFO [RS_OPEN_META-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-27T17:37:58,183 DEBUG [RS_OPEN_META-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-27T17:37:58,183 DEBUG [RS_OPEN_META-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T17:37:58,184 DEBUG [RS_OPEN_META-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-27T17:37:58,184 DEBUG [RS_OPEN_META-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-27T17:37:58,187 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T17:37:58,189 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T17:37:58,189 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:37:58,189 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T17:37:58,190 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T17:37:58,191 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T17:37:58,191 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:37:58,192 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T17:37:58,192 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T17:37:58,194 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T17:37:58,194 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:37:58,195 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T17:37:58,195 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T17:37:58,196 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T17:37:58,196 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:37:58,197 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T17:37:58,197 DEBUG [RS_OPEN_META-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T17:37:58,199 DEBUG [RS_OPEN_META-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/meta/1588230740 2024-11-27T17:37:58,202 DEBUG [RS_OPEN_META-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/meta/1588230740 2024-11-27T17:37:58,204 DEBUG [RS_OPEN_META-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T17:37:58,204 DEBUG [RS_OPEN_META-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T17:37:58,205 DEBUG [RS_OPEN_META-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T17:37:58,207 DEBUG [RS_OPEN_META-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T17:37:58,209 INFO [RS_OPEN_META-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62467892, jitterRate=-0.06915587186813354}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T17:37:58,209 DEBUG [RS_OPEN_META-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-27T17:37:58,210 DEBUG [RS_OPEN_META-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732729078184Writing region info on filesystem at 1732729078184Initializing all the Stores at 1732729078186 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732729078186Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732729078187 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732729078187Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732729078187Cleaning up temporary data from old regions at 1732729078204 (+17 ms)Running coprocessor post-open hooks at 1732729078209 (+5 ms)Region opened successfully at 1732729078210 (+1 ms) 2024-11-27T17:37:58,217 INFO [RS_OPEN_META-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732729078120 2024-11-27T17:37:58,228 DEBUG [RS_OPEN_META-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-27T17:37:58,229 INFO [RS_OPEN_META-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-27T17:37:58,230 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=8f0673442175,43899,1732729014292 2024-11-27T17:37:58,233 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8f0673442175,43899,1732729014292, state=OPEN 2024-11-27T17:37:58,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43899-0x1017d5e7cd30001, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T17:37:58,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46243-0x1017d5e7cd30000, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T17:37:58,316 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T17:37:58,316 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T17:37:58,317 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=8f0673442175,43899,1732729014292 2024-11-27T17:37:58,324 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-27T17:37:58,325 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=8f0673442175,43899,1732729014292 in 368 msec 2024-11-27T17:37:58,332 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-27T17:37:58,332 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1 mins, 0.473 sec 2024-11-27T17:37:58,334 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T17:37:58,334 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-27T17:37:58,351 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T17:37:58,352 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8f0673442175,43899,1732729014292, seqNum=-1] 2024-11-27T17:37:58,371 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T17:37:58,374 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51883, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T17:37:58,394 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1 mins, 1.188 sec 2024-11-27T17:37:58,394 INFO [master/8f0673442175:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732729078394, completionTime=-1 2024-11-27T17:37:58,396 INFO [master/8f0673442175:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-27T17:37:58,397 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T17:37:58,421 INFO [master/8f0673442175:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-27T17:37:58,421 INFO [master/8f0673442175:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732729138421 2024-11-27T17:37:58,422 INFO [master/8f0673442175:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732729198422 2024-11-27T17:37:58,422 INFO [master/8f0673442175:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-11-27T17:37:58,423 INFO [master/8f0673442175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8f0673442175,46243,1732729013300-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T17:37:58,424 INFO [master/8f0673442175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8f0673442175,46243,1732729013300-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T17:37:58,424 INFO [master/8f0673442175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8f0673442175,46243,1732729013300-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T17:37:58,425 INFO [master/8f0673442175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-8f0673442175:46243, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T17:37:58,426 INFO [master/8f0673442175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-27T17:37:58,426 INFO [master/8f0673442175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-27T17:37:58,433 DEBUG [master/8f0673442175:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-27T17:37:58,453 INFO [master/8f0673442175:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 63.843sec 2024-11-27T17:37:58,454 INFO [master/8f0673442175:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-27T17:37:58,455 INFO [master/8f0673442175:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-27T17:37:58,456 INFO [master/8f0673442175:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-27T17:37:58,457 INFO [master/8f0673442175:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-27T17:37:58,457 INFO [master/8f0673442175:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-27T17:37:58,458 INFO [master/8f0673442175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8f0673442175,46243,1732729013300-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T17:37:58,458 INFO [master/8f0673442175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8f0673442175,46243,1732729013300-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-27T17:37:58,467 DEBUG [master/8f0673442175:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-27T17:37:58,469 INFO [master/8f0673442175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-27T17:37:58,469 INFO [master/8f0673442175:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8f0673442175,46243,1732729013300-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T17:37:58,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f875e9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T17:37:58,481 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-27T17:37:58,481 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-27T17:37:58,484 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8f0673442175,46243,-1 for getting cluster id 2024-11-27T17:37:58,486 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T17:37:58,493 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1784d3b5-6fc6-48b8-ae1a-00e4efc5cff5' 2024-11-27T17:37:58,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T17:37:58,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1784d3b5-6fc6-48b8-ae1a-00e4efc5cff5" 2024-11-27T17:37:58,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6162a282, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T17:37:58,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8f0673442175,46243,-1] 2024-11-27T17:37:58,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T17:37:58,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T17:37:58,504 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T17:37:58,506 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a7ec3ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T17:37:58,507 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T17:37:58,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8f0673442175,43899,1732729014292, seqNum=-1] 2024-11-27T17:37:58,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T17:37:58,517 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T17:37:58,536 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=8f0673442175,46243,1732729013300 2024-11-27T17:37:58,539 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fb7c601, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T17:37:58,539 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8f0673442175,46243,-1 for getting cluster id 2024-11-27T17:37:58,539 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T17:37:58,541 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1784d3b5-6fc6-48b8-ae1a-00e4efc5cff5' 2024-11-27T17:37:58,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T17:37:58,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1784d3b5-6fc6-48b8-ae1a-00e4efc5cff5" 2024-11-27T17:37:58,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7db8c243, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T17:37:58,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8f0673442175,46243,-1] 2024-11-27T17:37:58,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T17:37:58,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T17:37:58,545 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T17:37:58,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43134d30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T17:37:58,561 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=208, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=118, ProcessCount=11, AvailableMemoryMB=10098 2024-11-27T17:37:58,568 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T17:37:58,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 8f0673442175,46243,1732729013300 2024-11-27T17:37:58,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1e5abfd9 2024-11-27T17:37:58,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T17:37:58,580 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T17:37:58,602 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T17:37:58,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync 2024-11-27T17:37:58,615 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T17:37:58,618 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextExceptionAsync" procId is: 4 2024-11-27T17:37:58,619 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:37:58,622 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T17:37:58,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T17:37:58,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741835_1011 (size=428) 2024-11-27T17:37:59,048 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a46374dfd125bdd1346773a9c685bf28, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732729078582.a46374dfd125bdd1346773a9c685bf28.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextExceptionAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f 2024-11-27T17:37:59,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741836_1012 (size=79) 2024-11-27T17:37:59,462 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732729078582.a46374dfd125bdd1346773a9c685bf28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T17:37:59,463 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1722): Closing a46374dfd125bdd1346773a9c685bf28, disabling compactions & flushes 2024-11-27T17:37:59,463 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732729078582.a46374dfd125bdd1346773a9c685bf28. 2024-11-27T17:37:59,463 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732729078582.a46374dfd125bdd1346773a9c685bf28. 2024-11-27T17:37:59,463 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732729078582.a46374dfd125bdd1346773a9c685bf28. after waiting 0 ms 2024-11-27T17:37:59,464 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732729078582.a46374dfd125bdd1346773a9c685bf28. 2024-11-27T17:37:59,464 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732729078582.a46374dfd125bdd1346773a9c685bf28. 2024-11-27T17:37:59,464 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextExceptionAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for a46374dfd125bdd1346773a9c685bf28: Waiting for close lock at 1732729079463Disabling compacts and flushes for region at 1732729079463Disabling writes for close at 1732729079464 (+1 ms)Writing region close event to WAL at 1732729079464Closed at 1732729079464 2024-11-27T17:37:59,470 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T17:37:59,476 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextExceptionAsync,,1732729078582.a46374dfd125bdd1346773a9c685bf28.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732729079470"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732729079470"}]},"ts":"1732729079470"} 2024-11-27T17:37:59,481 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T17:37:59,483 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T17:37:59,485 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732729079483"}]},"ts":"1732729079483"} 2024-11-27T17:37:59,490 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLING in hbase:meta 2024-11-27T17:37:59,492 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=a46374dfd125bdd1346773a9c685bf28, ASSIGN}] 2024-11-27T17:37:59,494 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=a46374dfd125bdd1346773a9c685bf28, ASSIGN 2024-11-27T17:37:59,496 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=a46374dfd125bdd1346773a9c685bf28, ASSIGN; state=OFFLINE, location=8f0673442175,43899,1732729014292; forceNewPlan=false, retain=false 2024-11-27T17:37:59,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T17:37:59,648 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a46374dfd125bdd1346773a9c685bf28, regionState=OPENING, regionLocation=8f0673442175,43899,1732729014292 2024-11-27T17:37:59,653 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=a46374dfd125bdd1346773a9c685bf28, ASSIGN because future has completed 2024-11-27T17:37:59,654 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a46374dfd125bdd1346773a9c685bf28, server=8f0673442175,43899,1732729014292}] 2024-11-27T17:37:59,815 INFO [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextExceptionAsync,,1732729078582.a46374dfd125bdd1346773a9c685bf28. 2024-11-27T17:37:59,815 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => a46374dfd125bdd1346773a9c685bf28, NAME => 'testRetryOutOfOrderScannerNextExceptionAsync,,1732729078582.a46374dfd125bdd1346773a9c685bf28.', STARTKEY => '', ENDKEY => ''} 2024-11-27T17:37:59,816 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextExceptionAsync a46374dfd125bdd1346773a9c685bf28 2024-11-27T17:37:59,816 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextExceptionAsync,,1732729078582.a46374dfd125bdd1346773a9c685bf28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T17:37:59,816 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for a46374dfd125bdd1346773a9c685bf28 2024-11-27T17:37:59,816 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for a46374dfd125bdd1346773a9c685bf28 2024-11-27T17:37:59,819 INFO [StoreOpener-a46374dfd125bdd1346773a9c685bf28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region a46374dfd125bdd1346773a9c685bf28 2024-11-27T17:37:59,821 INFO [StoreOpener-a46374dfd125bdd1346773a9c685bf28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a46374dfd125bdd1346773a9c685bf28 columnFamilyName testFamily 2024-11-27T17:37:59,821 DEBUG [StoreOpener-a46374dfd125bdd1346773a9c685bf28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:37:59,822 INFO [StoreOpener-a46374dfd125bdd1346773a9c685bf28-1 {}] regionserver.HStore(327): Store=a46374dfd125bdd1346773a9c685bf28/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T17:37:59,822 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for a46374dfd125bdd1346773a9c685bf28 2024-11-27T17:37:59,824 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testRetryOutOfOrderScannerNextExceptionAsync/a46374dfd125bdd1346773a9c685bf28 2024-11-27T17:37:59,824 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testRetryOutOfOrderScannerNextExceptionAsync/a46374dfd125bdd1346773a9c685bf28 2024-11-27T17:37:59,825 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for a46374dfd125bdd1346773a9c685bf28 2024-11-27T17:37:59,825 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for a46374dfd125bdd1346773a9c685bf28 2024-11-27T17:37:59,828 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for a46374dfd125bdd1346773a9c685bf28 2024-11-27T17:37:59,831 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testRetryOutOfOrderScannerNextExceptionAsync/a46374dfd125bdd1346773a9c685bf28/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T17:37:59,832 INFO [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened a46374dfd125bdd1346773a9c685bf28; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71491777, jitterRate=0.06531049311161041}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T17:37:59,832 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a46374dfd125bdd1346773a9c685bf28 2024-11-27T17:37:59,833 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for a46374dfd125bdd1346773a9c685bf28: Running coprocessor pre-open hook at 1732729079816Writing region info on filesystem at 1732729079816Initializing all the Stores at 1732729079818 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732729079818Cleaning up temporary data from old regions at 1732729079825 (+7 ms)Running coprocessor post-open hooks at 1732729079832 (+7 ms)Region opened successfully at 1732729079833 (+1 ms) 2024-11-27T17:37:59,836 INFO [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextExceptionAsync,,1732729078582.a46374dfd125bdd1346773a9c685bf28., pid=6, masterSystemTime=1732729079809 2024-11-27T17:37:59,839 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextExceptionAsync,,1732729078582.a46374dfd125bdd1346773a9c685bf28. 2024-11-27T17:37:59,839 INFO [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextExceptionAsync,,1732729078582.a46374dfd125bdd1346773a9c685bf28. 2024-11-27T17:37:59,840 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=a46374dfd125bdd1346773a9c685bf28, regionState=OPEN, openSeqNum=2, regionLocation=8f0673442175,43899,1732729014292 2024-11-27T17:37:59,845 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure a46374dfd125bdd1346773a9c685bf28, server=8f0673442175,43899,1732729014292 because future has completed 2024-11-27T17:37:59,851 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-27T17:37:59,851 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure a46374dfd125bdd1346773a9c685bf28, server=8f0673442175,43899,1732729014292 in 193 msec 2024-11-27T17:37:59,854 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-27T17:37:59,854 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextExceptionAsync, region=a46374dfd125bdd1346773a9c685bf28, ASSIGN in 359 msec 2024-11-27T17:37:59,856 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T17:37:59,856 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextExceptionAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732729079856"}]},"ts":"1732729079856"} 2024-11-27T17:37:59,859 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextExceptionAsync, state=ENABLED in hbase:meta 2024-11-27T17:37:59,860 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T17:37:59,863 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextExceptionAsync in 1.2540 sec 2024-11-27T17:38:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T17:38:01,665 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextExceptionAsync completed 2024-11-27T17:38:01,665 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextExceptionAsync get assigned. Timeout = 60000ms 2024-11-27T17:38:01,666 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T17:38:01,671 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned to meta. Checking AM states. 2024-11-27T17:38:01,672 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T17:38:01,672 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextExceptionAsync assigned. 2024-11-27T17:38:01,674 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T17:38:01,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8f0673442175,43899,1732729014292, seqNum=-1] 2024-11-27T17:38:01,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T17:38:01,679 INFO [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T17:38:01,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextExceptionAsync', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextExceptionAsync,,1732729078582.a46374dfd125bdd1346773a9c685bf28., hostname=8f0673442175,43899,1732729014292, seqNum=2] 2024-11-27T17:38:01,700 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-27T17:38:01,703 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextExceptionAsync,, stopping at row=testRetryOutOfOrderScannerNextExceptionAsync ,, for max=2147483647 with caching=100 2024-11-27T17:38:01,708 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-27T17:38:01,712 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-27T17:38:01,714 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-27T17:38:01,715 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43899 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=8336541563169538056 2024-11-27T17:38:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43899 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:42734 deadline: 1732729084713, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-27T17:38:01,724 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-27T17:38:01,726 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-27T17:38:01,727 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-27T17:38:01,729 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-27T17:38:01,730 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43899 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=8336541563169538058 2024-11-27T17:38:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43899 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:42734 deadline: 1732729084729, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-27T17:38:01,734 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-27T17:38:01,735 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43899 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=8336541563169538059 2024-11-27T17:38:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43899 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:42734 deadline: 1732729084734, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-27T17:38:01,739 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-27T17:38:01,740 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43899 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=8336541563169538060 2024-11-27T17:38:01,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43899 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:42734 deadline: 1732729084739, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-27T17:38:01,744 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-27T17:38:01,753 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextExceptionAsync Thread=215 (was 208) Potentially hanging thread: RS_OPEN_REGION-regionserver/8f0673442175:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1400453643_22 at /127.0.0.1:43544 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=449 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=132 (was 118) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10056 (was 10098) 2024-11-27T17:38:01,760 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=215, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=132, ProcessCount=11, AvailableMemoryMB=10056 2024-11-27T17:38:01,763 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46243 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T17:38:01,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46243 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner 2024-11-27T17:38:01,767 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T17:38:01,767 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:38:01,767 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46243 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScanner" procId is: 7 2024-11-27T17:38:01,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T17:38:01,769 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T17:38:01,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741837_1013 (size=414) 2024-11-27T17:38:02,187 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 34d7236d8a6e21ffac402f86622f6a1d, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732729081763.34d7236d8a6e21ffac402f86622f6a1d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f 2024-11-27T17:38:02,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741838_1014 (size=65) 2024-11-27T17:38:02,598 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732729081763.34d7236d8a6e21ffac402f86622f6a1d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T17:38:02,598 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 34d7236d8a6e21ffac402f86622f6a1d, disabling compactions & flushes 2024-11-27T17:38:02,598 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732729081763.34d7236d8a6e21ffac402f86622f6a1d. 2024-11-27T17:38:02,598 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732729081763.34d7236d8a6e21ffac402f86622f6a1d. 2024-11-27T17:38:02,598 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732729081763.34d7236d8a6e21ffac402f86622f6a1d. after waiting 0 ms 2024-11-27T17:38:02,599 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732729081763.34d7236d8a6e21ffac402f86622f6a1d. 2024-11-27T17:38:02,599 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732729081763.34d7236d8a6e21ffac402f86622f6a1d. 2024-11-27T17:38:02,599 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 34d7236d8a6e21ffac402f86622f6a1d: Waiting for close lock at 1732729082598Disabling compacts and flushes for region at 1732729082598Disabling writes for close at 1732729082599 (+1 ms)Writing region close event to WAL at 1732729082599Closed at 1732729082599 2024-11-27T17:38:02,602 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T17:38:02,603 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScanner,,1732729081763.34d7236d8a6e21ffac402f86622f6a1d.","families":{"info":[{"qualifier":"regioninfo","vlen":64,"tag":[],"timestamp":"1732729082603"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732729082603"}]},"ts":"1732729082603"} 2024-11-27T17:38:02,608 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T17:38:02,610 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T17:38:02,610 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732729082610"}]},"ts":"1732729082610"} 2024-11-27T17:38:02,613 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-27T17:38:02,613 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=34d7236d8a6e21ffac402f86622f6a1d, ASSIGN}] 2024-11-27T17:38:02,615 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=34d7236d8a6e21ffac402f86622f6a1d, ASSIGN 2024-11-27T17:38:02,617 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=34d7236d8a6e21ffac402f86622f6a1d, ASSIGN; state=OFFLINE, location=8f0673442175,43899,1732729014292; forceNewPlan=false, retain=false 2024-11-27T17:38:02,768 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=34d7236d8a6e21ffac402f86622f6a1d, regionState=OPENING, regionLocation=8f0673442175,43899,1732729014292 2024-11-27T17:38:02,778 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=34d7236d8a6e21ffac402f86622f6a1d, ASSIGN because future has completed 2024-11-27T17:38:02,780 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 34d7236d8a6e21ffac402f86622f6a1d, server=8f0673442175,43899,1732729014292}] 2024-11-27T17:38:02,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T17:38:02,942 INFO [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScanner,,1732729081763.34d7236d8a6e21ffac402f86622f6a1d. 2024-11-27T17:38:02,942 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 34d7236d8a6e21ffac402f86622f6a1d, NAME => 'hbase:testMetaScanTimeoutOnOpenScanner,,1732729081763.34d7236d8a6e21ffac402f86622f6a1d.', STARTKEY => '', ENDKEY => ''} 2024-11-27T17:38:02,943 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScanner 34d7236d8a6e21ffac402f86622f6a1d 2024-11-27T17:38:02,943 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScanner,,1732729081763.34d7236d8a6e21ffac402f86622f6a1d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T17:38:02,943 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 34d7236d8a6e21ffac402f86622f6a1d 2024-11-27T17:38:02,943 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 34d7236d8a6e21ffac402f86622f6a1d 2024-11-27T17:38:02,946 INFO [StoreOpener-34d7236d8a6e21ffac402f86622f6a1d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 34d7236d8a6e21ffac402f86622f6a1d 2024-11-27T17:38:02,948 INFO [StoreOpener-34d7236d8a6e21ffac402f86622f6a1d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34d7236d8a6e21ffac402f86622f6a1d columnFamilyName testFamily 2024-11-27T17:38:02,948 DEBUG [StoreOpener-34d7236d8a6e21ffac402f86622f6a1d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:38:02,949 INFO [StoreOpener-34d7236d8a6e21ffac402f86622f6a1d-1 {}] regionserver.HStore(327): Store=34d7236d8a6e21ffac402f86622f6a1d/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T17:38:02,950 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 34d7236d8a6e21ffac402f86622f6a1d 2024-11-27T17:38:02,951 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnOpenScanner/34d7236d8a6e21ffac402f86622f6a1d 2024-11-27T17:38:02,952 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnOpenScanner/34d7236d8a6e21ffac402f86622f6a1d 2024-11-27T17:38:02,953 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 34d7236d8a6e21ffac402f86622f6a1d 2024-11-27T17:38:02,953 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 34d7236d8a6e21ffac402f86622f6a1d 2024-11-27T17:38:02,956 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 34d7236d8a6e21ffac402f86622f6a1d 2024-11-27T17:38:02,960 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnOpenScanner/34d7236d8a6e21ffac402f86622f6a1d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T17:38:02,961 INFO [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1114): Opened 34d7236d8a6e21ffac402f86622f6a1d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70595559, jitterRate=0.051955804228782654}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T17:38:02,961 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 34d7236d8a6e21ffac402f86622f6a1d 2024-11-27T17:38:02,963 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 34d7236d8a6e21ffac402f86622f6a1d: Running coprocessor pre-open hook at 1732729082943Writing region info on filesystem at 1732729082943Initializing all the Stores at 1732729082945 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732729082945Cleaning up temporary data from old regions at 1732729082953 (+8 ms)Running coprocessor post-open hooks at 1732729082961 (+8 ms)Region opened successfully at 1732729082963 (+2 ms) 2024-11-27T17:38:02,964 INFO [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScanner,,1732729081763.34d7236d8a6e21ffac402f86622f6a1d., pid=9, masterSystemTime=1732729082934 2024-11-27T17:38:02,968 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScanner,,1732729081763.34d7236d8a6e21ffac402f86622f6a1d. 2024-11-27T17:38:02,968 INFO [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=9}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScanner,,1732729081763.34d7236d8a6e21ffac402f86622f6a1d. 2024-11-27T17:38:02,970 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=34d7236d8a6e21ffac402f86622f6a1d, regionState=OPEN, openSeqNum=2, regionLocation=8f0673442175,43899,1732729014292 2024-11-27T17:38:02,973 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 34d7236d8a6e21ffac402f86622f6a1d, server=8f0673442175,43899,1732729014292 because future has completed 2024-11-27T17:38:02,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-27T17:38:02,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 34d7236d8a6e21ffac402f86622f6a1d, server=8f0673442175,43899,1732729014292 in 197 msec 2024-11-27T17:38:02,984 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-27T17:38:02,984 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScanner, region=34d7236d8a6e21ffac402f86622f6a1d, ASSIGN in 367 msec 2024-11-27T17:38:02,986 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T17:38:02,986 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732729082986"}]},"ts":"1732729082986"} 2024-11-27T17:38:02,990 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-27T17:38:02,991 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=7, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T17:38:02,995 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScanner in 1.2270 sec 2024-11-27T17:38:03,724 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T17:38:03,751 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextExceptionAsync' 2024-11-27T17:38:03,753 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-27T17:38:03,754 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScanner' 2024-11-27T17:38:03,755 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-27T17:38:03,755 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-27T17:38:03,756 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T17:38:03,756 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-27T17:38:03,756 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync 2024-11-27T17:38:03,756 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextExceptionAsync Metrics about Tables on a single HBase RegionServer 2024-11-27T17:38:03,757 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner 2024-11-27T17:38:03,757 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-27T17:38:04,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T17:38:04,804 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScanner completed 2024-11-27T17:38:04,804 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-27T17:38:04,805 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T17:38:04,814 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-27T17:38:04,814 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T17:38:04,814 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScanner assigned. 2024-11-27T17:38:04,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScanner,,1732729081763.34d7236d8a6e21ffac402f86622f6a1d., hostname=8f0673442175,43899,1732729014292, seqNum=2] 2024-11-27T17:38:04,832 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-27T17:38:04,832 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScanner,, stopping at row=hbase:testMetaScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-27T17:38:04,835 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-27T17:38:04,837 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43899 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-27T17:38:10,856 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732729081763.34d7236d8a6e21ffac402f86622f6a1d., hostname=8f0673442175,43899,1732729014292, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScanner,,1732729081763.34d7236d8a6e21ffac402f86622f6a1d., hostname=8f0673442175,43899,1732729014292, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=8f0673442175:43899 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6018ms, rpcTimeout=5999ms 2024-11-27T17:38:10,857 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScanner,,1732729081763.34d7236d8a6e21ffac402f86622f6a1d., hostname=8f0673442175,43899,1732729014292, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=28,methodName=Scan], waitTime=6018ms, rpcTimeout=5999ms 2024-11-27T17:38:10,857 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScanner,,1732729081763.34d7236d8a6e21ffac402f86622f6a1d., hostname=8f0673442175,43899,1732729014292, seqNum=2 from cache 2024-11-27T17:38:10,869 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScanner Thread=218 (was 215) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=128 (was 132), ProcessCount=11 (was 11), AvailableMemoryMB=10043 (was 10056) 2024-11-27T17:38:10,877 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=218, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=128, ProcessCount=11, AvailableMemoryMB=10043 2024-11-27T17:38:10,880 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46243 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T17:38:10,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46243 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync 2024-11-27T17:38:10,885 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T17:38:10,885 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:38:10,885 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46243 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNextAsync" procId is: 10 2024-11-27T17:38:10,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-27T17:38:10,887 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T17:38:10,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741839_1015 (size=412) 2024-11-27T17:38:10,899 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f048b20286d5f531019fd9a36448940a, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732729090880.f048b20286d5f531019fd9a36448940a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f 2024-11-27T17:38:10,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741840_1016 (size=63) 2024-11-27T17:38:11,313 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732729090880.f048b20286d5f531019fd9a36448940a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T17:38:11,313 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing f048b20286d5f531019fd9a36448940a, disabling compactions & flushes 2024-11-27T17:38:11,313 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732729090880.f048b20286d5f531019fd9a36448940a. 2024-11-27T17:38:11,314 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732729090880.f048b20286d5f531019fd9a36448940a. 2024-11-27T17:38:11,314 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732729090880.f048b20286d5f531019fd9a36448940a. after waiting 0 ms 2024-11-27T17:38:11,314 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732729090880.f048b20286d5f531019fd9a36448940a. 2024-11-27T17:38:11,314 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732729090880.f048b20286d5f531019fd9a36448940a. 2024-11-27T17:38:11,315 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for f048b20286d5f531019fd9a36448940a: Waiting for close lock at 1732729091313Disabling compacts and flushes for region at 1732729091313Disabling writes for close at 1732729091314 (+1 ms)Writing region close event to WAL at 1732729091314Closed at 1732729091314 2024-11-27T17:38:11,317 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T17:38:11,318 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNextAsync,,1732729090880.f048b20286d5f531019fd9a36448940a.","families":{"info":[{"qualifier":"regioninfo","vlen":62,"tag":[],"timestamp":"1732729091318"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732729091318"}]},"ts":"1732729091318"} 2024-11-27T17:38:11,321 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T17:38:11,323 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T17:38:11,323 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732729091323"}]},"ts":"1732729091323"} 2024-11-27T17:38:11,327 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-27T17:38:11,327 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=f048b20286d5f531019fd9a36448940a, ASSIGN}] 2024-11-27T17:38:11,330 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=f048b20286d5f531019fd9a36448940a, ASSIGN 2024-11-27T17:38:11,332 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=f048b20286d5f531019fd9a36448940a, ASSIGN; state=OFFLINE, location=8f0673442175,43899,1732729014292; forceNewPlan=false, retain=false 2024-11-27T17:38:11,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 28, skipping over this response of 88 bytes 2024-11-27T17:38:11,484 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=f048b20286d5f531019fd9a36448940a, regionState=OPENING, regionLocation=8f0673442175,43899,1732729014292 2024-11-27T17:38:11,491 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=f048b20286d5f531019fd9a36448940a, ASSIGN because future has completed 2024-11-27T17:38:11,493 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure f048b20286d5f531019fd9a36448940a, server=8f0673442175,43899,1732729014292}] 2024-11-27T17:38:11,661 INFO [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNextAsync,,1732729090880.f048b20286d5f531019fd9a36448940a. 2024-11-27T17:38:11,661 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => f048b20286d5f531019fd9a36448940a, NAME => 'hbase:testMetaScanTimeoutOnNextAsync,,1732729090880.f048b20286d5f531019fd9a36448940a.', STARTKEY => '', ENDKEY => ''} 2024-11-27T17:38:11,661 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNextAsync f048b20286d5f531019fd9a36448940a 2024-11-27T17:38:11,662 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNextAsync,,1732729090880.f048b20286d5f531019fd9a36448940a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T17:38:11,662 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for f048b20286d5f531019fd9a36448940a 2024-11-27T17:38:11,662 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for f048b20286d5f531019fd9a36448940a 2024-11-27T17:38:11,664 INFO [StoreOpener-f048b20286d5f531019fd9a36448940a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region f048b20286d5f531019fd9a36448940a 2024-11-27T17:38:11,666 INFO [StoreOpener-f048b20286d5f531019fd9a36448940a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f048b20286d5f531019fd9a36448940a columnFamilyName testFamily 2024-11-27T17:38:11,666 DEBUG [StoreOpener-f048b20286d5f531019fd9a36448940a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:38:11,667 INFO [StoreOpener-f048b20286d5f531019fd9a36448940a-1 {}] regionserver.HStore(327): Store=f048b20286d5f531019fd9a36448940a/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T17:38:11,667 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for f048b20286d5f531019fd9a36448940a 2024-11-27T17:38:11,668 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnNextAsync/f048b20286d5f531019fd9a36448940a 2024-11-27T17:38:11,669 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnNextAsync/f048b20286d5f531019fd9a36448940a 2024-11-27T17:38:11,669 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for f048b20286d5f531019fd9a36448940a 2024-11-27T17:38:11,669 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for f048b20286d5f531019fd9a36448940a 2024-11-27T17:38:11,672 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for f048b20286d5f531019fd9a36448940a 2024-11-27T17:38:11,675 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnNextAsync/f048b20286d5f531019fd9a36448940a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T17:38:11,675 INFO [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1114): Opened f048b20286d5f531019fd9a36448940a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62782095, jitterRate=-0.06447388231754303}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T17:38:11,676 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f048b20286d5f531019fd9a36448940a 2024-11-27T17:38:11,676 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for f048b20286d5f531019fd9a36448940a: Running coprocessor pre-open hook at 1732729091662Writing region info on filesystem at 1732729091662Initializing all the Stores at 1732729091663 (+1 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732729091664 (+1 ms)Cleaning up temporary data from old regions at 1732729091670 (+6 ms)Running coprocessor post-open hooks at 1732729091676 (+6 ms)Region opened successfully at 1732729091676 2024-11-27T17:38:11,678 INFO [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNextAsync,,1732729090880.f048b20286d5f531019fd9a36448940a., pid=12, masterSystemTime=1732729091647 2024-11-27T17:38:11,681 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNextAsync,,1732729090880.f048b20286d5f531019fd9a36448940a. 2024-11-27T17:38:11,681 INFO [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=12}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNextAsync,,1732729090880.f048b20286d5f531019fd9a36448940a. 2024-11-27T17:38:11,682 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=f048b20286d5f531019fd9a36448940a, regionState=OPEN, openSeqNum=2, regionLocation=8f0673442175,43899,1732729014292 2024-11-27T17:38:11,685 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure f048b20286d5f531019fd9a36448940a, server=8f0673442175,43899,1732729014292 because future has completed 2024-11-27T17:38:11,690 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-27T17:38:11,690 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure f048b20286d5f531019fd9a36448940a, server=8f0673442175,43899,1732729014292 in 194 msec 2024-11-27T17:38:11,694 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-27T17:38:11,694 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNextAsync, region=f048b20286d5f531019fd9a36448940a, ASSIGN in 363 msec 2024-11-27T17:38:11,695 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T17:38:11,695 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732729091695"}]},"ts":"1732729091695"} 2024-11-27T17:38:11,698 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-27T17:38:11,699 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=10, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T17:38:11,702 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNextAsync in 818 msec 2024-11-27T17:38:11,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=10 2024-11-27T17:38:11,904 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNextAsync completed 2024-11-27T17:38:11,905 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-27T17:38:11,905 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T17:38:11,913 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-27T17:38:11,913 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T17:38:11,913 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNextAsync assigned. 2024-11-27T17:38:11,917 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNextAsync,,1732729090880.f048b20286d5f531019fd9a36448940a., hostname=8f0673442175,43899,1732729014292, seqNum=2] 2024-11-27T17:38:11,929 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-27T17:38:11,929 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNextAsync,, stopping at row=hbase:testMetaScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-27T17:38:11,932 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-27T17:38:11,934 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-27T17:38:11,936 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-27T17:38:11,937 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43899 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-27T17:38:13,676 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync 2024-11-27T17:38:13,676 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-27T17:38:15,757 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNextAsync' 2024-11-27T17:38:17,192 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T17:38:17,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43899 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '8336541563169538069' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-27T17:38:17,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43899 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:42734 deadline: 1732729103950, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '8336541563169538069' does not exist 2024-11-27T17:38:17,956 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 8f0673442175,43899,1732729014292 for closing scanner id = 8336541563169538069 for f048b20286d5f531019fd9a36448940a of hbase:testMetaScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '8336541563169538069' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T17:38:17,959 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNextAsync Thread=220 (was 218) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=118 (was 128), ProcessCount=11 (was 11), AvailableMemoryMB=10041 (was 10043) 2024-11-27T17:38:17,968 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=220, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=118, ProcessCount=11, AvailableMemoryMB=10040 2024-11-27T17:38:17,971 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T17:38:17,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException 2024-11-27T17:38:17,974 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T17:38:17,974 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:38:17,974 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testRetryOutOfOrderScannerNextException" procId is: 13 2024-11-27T17:38:17,975 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T17:38:17,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T17:38:17,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741841_1017 (size=423) 2024-11-27T17:38:18,393 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a82ab4b705fac222998b7c9c55b9d4a6, NAME => 'testRetryOutOfOrderScannerNextException,,1732729097970.a82ab4b705fac222998b7c9c55b9d4a6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRetryOutOfOrderScannerNextException', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f 2024-11-27T17:38:18,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741842_1018 (size=74) 2024-11-27T17:38:18,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 37, skipping over this response of 82 bytes 2024-11-27T17:38:18,808 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732729097970.a82ab4b705fac222998b7c9c55b9d4a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T17:38:18,809 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1722): Closing a82ab4b705fac222998b7c9c55b9d4a6, disabling compactions & flushes 2024-11-27T17:38:18,809 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732729097970.a82ab4b705fac222998b7c9c55b9d4a6. 2024-11-27T17:38:18,809 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732729097970.a82ab4b705fac222998b7c9c55b9d4a6. 2024-11-27T17:38:18,810 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732729097970.a82ab4b705fac222998b7c9c55b9d4a6. after waiting 0 ms 2024-11-27T17:38:18,810 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732729097970.a82ab4b705fac222998b7c9c55b9d4a6. 2024-11-27T17:38:18,810 INFO [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732729097970.a82ab4b705fac222998b7c9c55b9d4a6. 2024-11-27T17:38:18,810 DEBUG [RegionOpenAndInit-testRetryOutOfOrderScannerNextException-pool-0 {}] regionserver.HRegion(1676): Region close journal for a82ab4b705fac222998b7c9c55b9d4a6: Waiting for close lock at 1732729098809Disabling compacts and flushes for region at 1732729098809Disabling writes for close at 1732729098810 (+1 ms)Writing region close event to WAL at 1732729098810Closed at 1732729098810 2024-11-27T17:38:18,813 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T17:38:18,814 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testRetryOutOfOrderScannerNextException,,1732729097970.a82ab4b705fac222998b7c9c55b9d4a6.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732729098813"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732729098813"}]},"ts":"1732729098813"} 2024-11-27T17:38:18,817 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T17:38:18,818 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T17:38:18,818 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732729098818"}]},"ts":"1732729098818"} 2024-11-27T17:38:18,821 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLING in hbase:meta 2024-11-27T17:38:18,822 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=a82ab4b705fac222998b7c9c55b9d4a6, ASSIGN}] 2024-11-27T17:38:18,824 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=a82ab4b705fac222998b7c9c55b9d4a6, ASSIGN 2024-11-27T17:38:18,825 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=a82ab4b705fac222998b7c9c55b9d4a6, ASSIGN; state=OFFLINE, location=8f0673442175,43899,1732729014292; forceNewPlan=false, retain=false 2024-11-27T17:38:18,977 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=a82ab4b705fac222998b7c9c55b9d4a6, regionState=OPENING, regionLocation=8f0673442175,43899,1732729014292 2024-11-27T17:38:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T17:38:18,987 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=a82ab4b705fac222998b7c9c55b9d4a6, ASSIGN because future has completed 2024-11-27T17:38:18,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure a82ab4b705fac222998b7c9c55b9d4a6, server=8f0673442175,43899,1732729014292}] 2024-11-27T17:38:19,154 INFO [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(132): Open testRetryOutOfOrderScannerNextException,,1732729097970.a82ab4b705fac222998b7c9c55b9d4a6. 2024-11-27T17:38:19,155 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7752): Opening region: {ENCODED => a82ab4b705fac222998b7c9c55b9d4a6, NAME => 'testRetryOutOfOrderScannerNextException,,1732729097970.a82ab4b705fac222998b7c9c55b9d4a6.', STARTKEY => '', ENDKEY => ''} 2024-11-27T17:38:19,155 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testRetryOutOfOrderScannerNextException a82ab4b705fac222998b7c9c55b9d4a6 2024-11-27T17:38:19,155 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(898): Instantiated testRetryOutOfOrderScannerNextException,,1732729097970.a82ab4b705fac222998b7c9c55b9d4a6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T17:38:19,156 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7794): checking encryption for a82ab4b705fac222998b7c9c55b9d4a6 2024-11-27T17:38:19,156 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7797): checking classloading for a82ab4b705fac222998b7c9c55b9d4a6 2024-11-27T17:38:19,158 INFO [StoreOpener-a82ab4b705fac222998b7c9c55b9d4a6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region a82ab4b705fac222998b7c9c55b9d4a6 2024-11-27T17:38:19,161 INFO [StoreOpener-a82ab4b705fac222998b7c9c55b9d4a6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a82ab4b705fac222998b7c9c55b9d4a6 columnFamilyName testFamily 2024-11-27T17:38:19,161 DEBUG [StoreOpener-a82ab4b705fac222998b7c9c55b9d4a6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:38:19,162 INFO [StoreOpener-a82ab4b705fac222998b7c9c55b9d4a6-1 {}] regionserver.HStore(327): Store=a82ab4b705fac222998b7c9c55b9d4a6/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T17:38:19,162 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1038): replaying wal for a82ab4b705fac222998b7c9c55b9d4a6 2024-11-27T17:38:19,164 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testRetryOutOfOrderScannerNextException/a82ab4b705fac222998b7c9c55b9d4a6 2024-11-27T17:38:19,165 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testRetryOutOfOrderScannerNextException/a82ab4b705fac222998b7c9c55b9d4a6 2024-11-27T17:38:19,165 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1048): stopping wal replay for a82ab4b705fac222998b7c9c55b9d4a6 2024-11-27T17:38:19,165 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1060): Cleaning up temporary data for a82ab4b705fac222998b7c9c55b9d4a6 2024-11-27T17:38:19,168 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): writing seq id for a82ab4b705fac222998b7c9c55b9d4a6 2024-11-27T17:38:19,172 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testRetryOutOfOrderScannerNextException/a82ab4b705fac222998b7c9c55b9d4a6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T17:38:19,173 INFO [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1114): Opened a82ab4b705fac222998b7c9c55b9d4a6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65802438, jitterRate=-0.019467264413833618}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T17:38:19,173 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a82ab4b705fac222998b7c9c55b9d4a6 2024-11-27T17:38:19,174 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1006): Region open journal for a82ab4b705fac222998b7c9c55b9d4a6: Running coprocessor pre-open hook at 1732729099156Writing region info on filesystem at 1732729099156Initializing all the Stores at 1732729099158 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732729099158Cleaning up temporary data from old regions at 1732729099165 (+7 ms)Running coprocessor post-open hooks at 1732729099173 (+8 ms)Region opened successfully at 1732729099174 (+1 ms) 2024-11-27T17:38:19,175 INFO [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2236): Post open deploy tasks for testRetryOutOfOrderScannerNextException,,1732729097970.a82ab4b705fac222998b7c9c55b9d4a6., pid=15, masterSystemTime=1732729099142 2024-11-27T17:38:19,178 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2266): Finished post open deploy task for testRetryOutOfOrderScannerNextException,,1732729097970.a82ab4b705fac222998b7c9c55b9d4a6. 2024-11-27T17:38:19,179 INFO [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(153): Opened testRetryOutOfOrderScannerNextException,,1732729097970.a82ab4b705fac222998b7c9c55b9d4a6. 2024-11-27T17:38:19,180 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=a82ab4b705fac222998b7c9c55b9d4a6, regionState=OPEN, openSeqNum=2, regionLocation=8f0673442175,43899,1732729014292 2024-11-27T17:38:19,183 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure a82ab4b705fac222998b7c9c55b9d4a6, server=8f0673442175,43899,1732729014292 because future has completed 2024-11-27T17:38:19,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-27T17:38:19,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure a82ab4b705fac222998b7c9c55b9d4a6, server=8f0673442175,43899,1732729014292 in 197 msec 2024-11-27T17:38:19,191 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-27T17:38:19,192 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testRetryOutOfOrderScannerNextException, region=a82ab4b705fac222998b7c9c55b9d4a6, ASSIGN in 367 msec 2024-11-27T17:38:19,193 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T17:38:19,193 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testRetryOutOfOrderScannerNextException","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732729099193"}]},"ts":"1732729099193"} 2024-11-27T17:38:19,196 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testRetryOutOfOrderScannerNextException, state=ENABLED in hbase:meta 2024-11-27T17:38:19,197 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testRetryOutOfOrderScannerNextException execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T17:38:19,200 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; CreateTableProcedure table=testRetryOutOfOrderScannerNextException in 1.2260 sec 2024-11-27T17:38:21,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T17:38:21,003 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testRetryOutOfOrderScannerNextException completed 2024-11-27T17:38:21,003 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testRetryOutOfOrderScannerNextException get assigned. Timeout = 60000ms 2024-11-27T17:38:21,003 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T17:38:21,010 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testRetryOutOfOrderScannerNextException assigned to meta. Checking AM states. 2024-11-27T17:38:21,010 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T17:38:21,010 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testRetryOutOfOrderScannerNextException assigned. 2024-11-27T17:38:21,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testRetryOutOfOrderScannerNextException', row='row-0', locateType=CURRENT is [region=testRetryOutOfOrderScannerNextException,,1732729097970.a82ab4b705fac222998b7c9c55b9d4a6., hostname=8f0673442175,43899,1732729014292, seqNum=2] 2024-11-27T17:38:21,027 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-27T17:38:21,027 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testRetryOutOfOrderScannerNextException,, stopping at row=testRetryOutOfOrderScannerNextException ,, for max=2147483647 with caching=100 2024-11-27T17:38:21,030 INFO [Time-limited test {}] client.TestClientScannerTimeouts(252): Opening scanner, expecting no errors from first next() call from openScanner response 2024-11-27T17:38:21,032 INFO [Time-limited test {}] client.TestClientScannerTimeouts(259): Making first next() RPC, expecting no errors for seqNo 0 2024-11-27T17:38:21,034 INFO [Time-limited test {}] client.TestClientScannerTimeouts(264): Making second next() RPC, expecting OutOfOrderScannerNextException and appropriate retry 2024-11-27T17:38:21,034 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43899 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=8336541563169538073 2024-11-27T17:38:21,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43899 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:42734 deadline: 1732729104033, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-27T17:38:21,037 INFO [Time-limited test {}] client.TestClientScannerTimeouts(274): Finishing scan, expecting no errors 2024-11-27T17:38:21,038 INFO [Time-limited test {}] client.TestClientScannerTimeouts(279): Testing always throw exception 2024-11-27T17:38:21,039 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-27T17:38:21,040 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-27T17:38:21,041 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43899 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=1, tableScannerId=8336541563169538075 2024-11-27T17:38:21,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43899 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:42734 deadline: 1732729104040, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-27T17:38:21,044 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-27T17:38:21,045 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43899 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=2, tableScannerId=8336541563169538076 2024-11-27T17:38:21,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43899 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:42734 deadline: 1732729104043, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-27T17:38:21,048 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-27T17:38:21,048 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43899 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(438): THROWING exception, tryNumber=3, tableScannerId=8336541563169538077 2024-11-27T17:38:21,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43899 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Scan size: 43 connection: 172.17.0.3:42734 deadline: 1732729104047, exception=org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException 2024-11-27T17:38:21,051 INFO [Time-limited test {}] client.TestClientScannerTimeouts(288): Calling scanner.next() 2024-11-27T17:38:21,059 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testRetryOutOfOrderScannerNextException Thread=223 (was 220) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1400453643_22 at /127.0.0.1:57272 [Waiting for operation #2] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=108 (was 118), ProcessCount=11 (was 11), AvailableMemoryMB=10037 (was 10040) 2024-11-27T17:38:21,067 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=223, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=108, ProcessCount=11, AvailableMemoryMB=10036 2024-11-27T17:38:21,071 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T17:38:21,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] procedure2.ProcedureExecutor(1139): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner 2024-11-27T17:38:21,074 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T17:38:21,074 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:38:21,074 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScanner" procId is: 16 2024-11-27T17:38:21,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-27T17:38:21,076 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T17:38:21,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741843_1019 (size=418) 2024-11-27T17:38:21,497 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3fd0994b61fc0961c034fe9da87dbbfd, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScanner', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f 2024-11-27T17:38:21,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741844_1020 (size=69) 2024-11-27T17:38:21,759 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testRetryOutOfOrderScannerNextException' 2024-11-27T17:38:21,912 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T17:38:21,913 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1722): Closing 3fd0994b61fc0961c034fe9da87dbbfd, disabling compactions & flushes 2024-11-27T17:38:21,913 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd. 2024-11-27T17:38:21,913 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd. 2024-11-27T17:38:21,913 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd. after waiting 0 ms 2024-11-27T17:38:21,913 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd. 2024-11-27T17:38:21,914 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd. 2024-11-27T17:38:21,914 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScanner-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3fd0994b61fc0961c034fe9da87dbbfd: Waiting for close lock at 1732729101913Disabling compacts and flushes for region at 1732729101913Disabling writes for close at 1732729101913Writing region close event to WAL at 1732729101914 (+1 ms)Closed at 1732729101914 2024-11-27T17:38:21,918 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T17:38:21,919 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd.","families":{"info":[{"qualifier":"regioninfo","vlen":68,"tag":[],"timestamp":"1732729101919"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732729101919"}]},"ts":"1732729101919"} 2024-11-27T17:38:21,924 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T17:38:21,926 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T17:38:21,926 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732729101926"}]},"ts":"1732729101926"} 2024-11-27T17:38:21,930 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLING in hbase:meta 2024-11-27T17:38:21,930 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=3fd0994b61fc0961c034fe9da87dbbfd, ASSIGN}] 2024-11-27T17:38:21,933 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=3fd0994b61fc0961c034fe9da87dbbfd, ASSIGN 2024-11-27T17:38:21,934 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=3fd0994b61fc0961c034fe9da87dbbfd, ASSIGN; state=OFFLINE, location=8f0673442175,43899,1732729014292; forceNewPlan=false, retain=false 2024-11-27T17:38:22,086 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=3fd0994b61fc0961c034fe9da87dbbfd, regionState=OPENING, regionLocation=8f0673442175,43899,1732729014292 2024-11-27T17:38:22,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-27T17:38:22,092 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=3fd0994b61fc0961c034fe9da87dbbfd, ASSIGN because future has completed 2024-11-27T17:38:22,093 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3fd0994b61fc0961c034fe9da87dbbfd, server=8f0673442175,43899,1732729014292}] 2024-11-27T17:38:22,260 INFO [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd. 2024-11-27T17:38:22,260 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7752): Opening region: {ENCODED => 3fd0994b61fc0961c034fe9da87dbbfd, NAME => 'testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd.', STARTKEY => '', ENDKEY => ''} 2024-11-27T17:38:22,261 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScanner 3fd0994b61fc0961c034fe9da87dbbfd 2024-11-27T17:38:22,261 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T17:38:22,261 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7794): checking encryption for 3fd0994b61fc0961c034fe9da87dbbfd 2024-11-27T17:38:22,261 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7797): checking classloading for 3fd0994b61fc0961c034fe9da87dbbfd 2024-11-27T17:38:22,264 INFO [StoreOpener-3fd0994b61fc0961c034fe9da87dbbfd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 3fd0994b61fc0961c034fe9da87dbbfd 2024-11-27T17:38:22,266 INFO [StoreOpener-3fd0994b61fc0961c034fe9da87dbbfd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3fd0994b61fc0961c034fe9da87dbbfd columnFamilyName testFamily 2024-11-27T17:38:22,266 DEBUG [StoreOpener-3fd0994b61fc0961c034fe9da87dbbfd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:38:22,267 INFO [StoreOpener-3fd0994b61fc0961c034fe9da87dbbfd-1 {}] regionserver.HStore(327): Store=3fd0994b61fc0961c034fe9da87dbbfd/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T17:38:22,267 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1038): replaying wal for 3fd0994b61fc0961c034fe9da87dbbfd 2024-11-27T17:38:22,268 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnOpenScanner/3fd0994b61fc0961c034fe9da87dbbfd 2024-11-27T17:38:22,269 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnOpenScanner/3fd0994b61fc0961c034fe9da87dbbfd 2024-11-27T17:38:22,270 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1048): stopping wal replay for 3fd0994b61fc0961c034fe9da87dbbfd 2024-11-27T17:38:22,270 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1060): Cleaning up temporary data for 3fd0994b61fc0961c034fe9da87dbbfd 2024-11-27T17:38:22,272 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): writing seq id for 3fd0994b61fc0961c034fe9da87dbbfd 2024-11-27T17:38:22,275 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnOpenScanner/3fd0994b61fc0961c034fe9da87dbbfd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T17:38:22,276 INFO [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1114): Opened 3fd0994b61fc0961c034fe9da87dbbfd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68265116, jitterRate=0.01722949743270874}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T17:38:22,276 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3fd0994b61fc0961c034fe9da87dbbfd 2024-11-27T17:38:22,277 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1006): Region open journal for 3fd0994b61fc0961c034fe9da87dbbfd: Running coprocessor pre-open hook at 1732729102261Writing region info on filesystem at 1732729102261Initializing all the Stores at 1732729102264 (+3 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732729102264Cleaning up temporary data from old regions at 1732729102270 (+6 ms)Running coprocessor post-open hooks at 1732729102276 (+6 ms)Region opened successfully at 1732729102277 (+1 ms) 2024-11-27T17:38:22,278 INFO [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd., pid=18, masterSystemTime=1732729102248 2024-11-27T17:38:22,281 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd. 2024-11-27T17:38:22,281 INFO [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd. 2024-11-27T17:38:22,282 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=3fd0994b61fc0961c034fe9da87dbbfd, regionState=OPEN, openSeqNum=2, regionLocation=8f0673442175,43899,1732729014292 2024-11-27T17:38:22,285 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3fd0994b61fc0961c034fe9da87dbbfd, server=8f0673442175,43899,1732729014292 because future has completed 2024-11-27T17:38:22,289 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-27T17:38:22,289 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure 3fd0994b61fc0961c034fe9da87dbbfd, server=8f0673442175,43899,1732729014292 in 193 msec 2024-11-27T17:38:22,292 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-27T17:38:22,292 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScanner, region=3fd0994b61fc0961c034fe9da87dbbfd, ASSIGN in 359 msec 2024-11-27T17:38:22,293 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T17:38:22,293 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScanner","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732729102293"}]},"ts":"1732729102293"} 2024-11-27T17:38:22,296 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScanner, state=ENABLED in hbase:meta 2024-11-27T17:38:22,297 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T17:38:22,300 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScanner in 1.2260 sec 2024-11-27T17:38:22,473 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T17:38:23,676 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException 2024-11-27T17:38:23,676 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRetryOutOfOrderScannerNextException Metrics about Tables on a single HBase RegionServer 2024-11-27T17:38:23,679 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner 2024-11-27T17:38:23,679 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScanner Metrics about Tables on a single HBase RegionServer 2024-11-27T17:38:24,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=16 2024-11-27T17:38:24,123 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScanner completed 2024-11-27T17:38:24,123 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScanner get assigned. Timeout = 60000ms 2024-11-27T17:38:24,124 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T17:38:24,133 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScanner assigned to meta. Checking AM states. 2024-11-27T17:38:24,133 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T17:38:24,133 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScanner assigned. 2024-11-27T17:38:24,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd., hostname=8f0673442175,43899,1732729014292, seqNum=2] 2024-11-27T17:38:24,147 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-27T17:38:24,147 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScanner,, stopping at row=testNormalScanTimeoutOnOpenScanner ,, for max=2147483647 with caching=100 2024-11-27T17:38:24,151 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-27T17:38:24,153 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43899 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-27T17:38:24,659 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T17:38:25,155 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd., hostname=8f0673442175,43899,1732729014292, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd., hostname=8f0673442175,43899,1732729014292, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=8f0673442175:43899 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1002ms, rpcTimeout=1000ms 2024-11-27T17:38:25,155 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd., hostname=8f0673442175,43899,1732729014292, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=67,methodName=Scan], waitTime=1002ms, rpcTimeout=1000ms 2024-11-27T17:38:25,155 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd., hostname=8f0673442175,43899,1732729014292, seqNum=2 from cache 2024-11-27T17:38:25,655 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 67, skipping over this response of 88 bytes 2024-11-27T17:38:26,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScanner', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd., hostname=8f0673442175,43899,1732729014292, seqNum=2] 2024-11-27T17:38:26,182 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43899 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-27T17:38:27,155 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd., hostname=8f0673442175,43899,1732729014292, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd., hostname=8f0673442175,43899,1732729014292, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=8f0673442175:43899 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=976ms, rpcTimeout=973ms 2024-11-27T17:38:27,156 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd., hostname=8f0673442175,43899,1732729014292, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=69,methodName=Scan], waitTime=976ms, rpcTimeout=973ms 2024-11-27T17:38:27,156 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd., hostname=8f0673442175,43899,1732729014292, seqNum=2 from cache 2024-11-27T17:38:27,174 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScanner Thread=222 (was 223), OpenFileDescriptor=448 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=116 (was 108) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10034 (was 10036) 2024-11-27T17:38:27,184 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=116, ProcessCount=11, AvailableMemoryMB=10033 2024-11-27T17:38:27,187 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T17:38:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync 2024-11-27T17:38:27,192 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T17:38:27,192 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:38:27,192 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNextAsync" procId is: 19 2024-11-27T17:38:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T17:38:27,194 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T17:38:27,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741845_1021 (size=416) 2024-11-27T17:38:27,611 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 87e3e592443f2e8ffc03e943933578f6, NAME => 'testNormalScanTimeoutOnNextAsync,,1732729107187.87e3e592443f2e8ffc03e943933578f6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNextAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f 2024-11-27T17:38:27,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741846_1022 (size=67) 2024-11-27T17:38:27,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 69, skipping over this response of 88 bytes 2024-11-27T17:38:27,763 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScanner' 2024-11-27T17:38:28,027 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732729107187.87e3e592443f2e8ffc03e943933578f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T17:38:28,027 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1722): Closing 87e3e592443f2e8ffc03e943933578f6, disabling compactions & flushes 2024-11-27T17:38:28,027 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732729107187.87e3e592443f2e8ffc03e943933578f6. 2024-11-27T17:38:28,027 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732729107187.87e3e592443f2e8ffc03e943933578f6. 2024-11-27T17:38:28,028 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732729107187.87e3e592443f2e8ffc03e943933578f6. after waiting 0 ms 2024-11-27T17:38:28,028 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732729107187.87e3e592443f2e8ffc03e943933578f6. 2024-11-27T17:38:28,028 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732729107187.87e3e592443f2e8ffc03e943933578f6. 2024-11-27T17:38:28,029 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNextAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 87e3e592443f2e8ffc03e943933578f6: Waiting for close lock at 1732729108027Disabling compacts and flushes for region at 1732729108027Disabling writes for close at 1732729108028 (+1 ms)Writing region close event to WAL at 1732729108028Closed at 1732729108028 2024-11-27T17:38:28,034 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T17:38:28,034 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNextAsync,,1732729107187.87e3e592443f2e8ffc03e943933578f6.","families":{"info":[{"qualifier":"regioninfo","vlen":66,"tag":[],"timestamp":"1732729108034"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732729108034"}]},"ts":"1732729108034"} 2024-11-27T17:38:28,037 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T17:38:28,039 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T17:38:28,039 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732729108039"}]},"ts":"1732729108039"} 2024-11-27T17:38:28,042 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLING in hbase:meta 2024-11-27T17:38:28,043 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=87e3e592443f2e8ffc03e943933578f6, ASSIGN}] 2024-11-27T17:38:28,045 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=87e3e592443f2e8ffc03e943933578f6, ASSIGN 2024-11-27T17:38:28,047 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=87e3e592443f2e8ffc03e943933578f6, ASSIGN; state=OFFLINE, location=8f0673442175,43899,1732729014292; forceNewPlan=false, retain=false 2024-11-27T17:38:28,198 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=87e3e592443f2e8ffc03e943933578f6, regionState=OPENING, regionLocation=8f0673442175,43899,1732729014292 2024-11-27T17:38:28,205 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=87e3e592443f2e8ffc03e943933578f6, ASSIGN because future has completed 2024-11-27T17:38:28,206 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87e3e592443f2e8ffc03e943933578f6, server=8f0673442175,43899,1732729014292}] 2024-11-27T17:38:28,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T17:38:28,373 INFO [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNextAsync,,1732729107187.87e3e592443f2e8ffc03e943933578f6. 2024-11-27T17:38:28,373 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 87e3e592443f2e8ffc03e943933578f6, NAME => 'testNormalScanTimeoutOnNextAsync,,1732729107187.87e3e592443f2e8ffc03e943933578f6.', STARTKEY => '', ENDKEY => ''} 2024-11-27T17:38:28,374 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNextAsync 87e3e592443f2e8ffc03e943933578f6 2024-11-27T17:38:28,374 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNextAsync,,1732729107187.87e3e592443f2e8ffc03e943933578f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T17:38:28,374 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 87e3e592443f2e8ffc03e943933578f6 2024-11-27T17:38:28,374 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 87e3e592443f2e8ffc03e943933578f6 2024-11-27T17:38:28,376 INFO [StoreOpener-87e3e592443f2e8ffc03e943933578f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 87e3e592443f2e8ffc03e943933578f6 2024-11-27T17:38:28,378 INFO [StoreOpener-87e3e592443f2e8ffc03e943933578f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87e3e592443f2e8ffc03e943933578f6 columnFamilyName testFamily 2024-11-27T17:38:28,378 DEBUG [StoreOpener-87e3e592443f2e8ffc03e943933578f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:38:28,379 INFO [StoreOpener-87e3e592443f2e8ffc03e943933578f6-1 {}] regionserver.HStore(327): Store=87e3e592443f2e8ffc03e943933578f6/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T17:38:28,379 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 87e3e592443f2e8ffc03e943933578f6 2024-11-27T17:38:28,381 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnNextAsync/87e3e592443f2e8ffc03e943933578f6 2024-11-27T17:38:28,381 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnNextAsync/87e3e592443f2e8ffc03e943933578f6 2024-11-27T17:38:28,382 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 87e3e592443f2e8ffc03e943933578f6 2024-11-27T17:38:28,382 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 87e3e592443f2e8ffc03e943933578f6 2024-11-27T17:38:28,384 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 87e3e592443f2e8ffc03e943933578f6 2024-11-27T17:38:28,388 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnNextAsync/87e3e592443f2e8ffc03e943933578f6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T17:38:28,388 INFO [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 87e3e592443f2e8ffc03e943933578f6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59886525, jitterRate=-0.10762123763561249}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T17:38:28,389 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 87e3e592443f2e8ffc03e943933578f6 2024-11-27T17:38:28,389 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 87e3e592443f2e8ffc03e943933578f6: Running coprocessor pre-open hook at 1732729108374Writing region info on filesystem at 1732729108374Initializing all the Stores at 1732729108376 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732729108376Cleaning up temporary data from old regions at 1732729108382 (+6 ms)Running coprocessor post-open hooks at 1732729108389 (+7 ms)Region opened successfully at 1732729108389 2024-11-27T17:38:28,391 INFO [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNextAsync,,1732729107187.87e3e592443f2e8ffc03e943933578f6., pid=21, masterSystemTime=1732729108360 2024-11-27T17:38:28,394 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNextAsync,,1732729107187.87e3e592443f2e8ffc03e943933578f6. 2024-11-27T17:38:28,394 INFO [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNextAsync,,1732729107187.87e3e592443f2e8ffc03e943933578f6. 2024-11-27T17:38:28,395 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=87e3e592443f2e8ffc03e943933578f6, regionState=OPEN, openSeqNum=2, regionLocation=8f0673442175,43899,1732729014292 2024-11-27T17:38:28,398 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87e3e592443f2e8ffc03e943933578f6, server=8f0673442175,43899,1732729014292 because future has completed 2024-11-27T17:38:28,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-27T17:38:28,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 87e3e592443f2e8ffc03e943933578f6, server=8f0673442175,43899,1732729014292 in 193 msec 2024-11-27T17:38:28,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-27T17:38:28,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNextAsync, region=87e3e592443f2e8ffc03e943933578f6, ASSIGN in 359 msec 2024-11-27T17:38:28,407 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T17:38:28,407 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNextAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732729108407"}]},"ts":"1732729108407"} 2024-11-27T17:38:28,410 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNextAsync, state=ENABLED in hbase:meta 2024-11-27T17:38:28,411 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T17:38:28,413 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNextAsync in 1.2240 sec 2024-11-27T17:38:30,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T17:38:30,224 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNextAsync completed 2024-11-27T17:38:30,225 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNextAsync get assigned. Timeout = 60000ms 2024-11-27T17:38:30,225 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T17:38:30,238 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNextAsync assigned to meta. Checking AM states. 2024-11-27T17:38:30,239 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T17:38:30,239 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNextAsync assigned. 2024-11-27T17:38:30,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNextAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNextAsync,,1732729107187.87e3e592443f2e8ffc03e943933578f6., hostname=8f0673442175,43899,1732729014292, seqNum=2] 2024-11-27T17:38:30,254 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-27T17:38:30,254 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNextAsync,, stopping at row=testNormalScanTimeoutOnNextAsync ,, for max=2147483647 with caching=100 2024-11-27T17:38:30,258 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-27T17:38:30,261 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-27T17:38:30,262 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-27T17:38:30,263 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43899 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-27T17:38:33,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43899 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '8336541563169538088' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-27T17:38:33,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43899 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:42734 deadline: 1732729114267, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '8336541563169538088' does not exist 2024-11-27T17:38:33,268 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 8f0673442175,43899,1732729014292 for closing scanner id = 8336541563169538088 for 87e3e592443f2e8ffc03e943933578f6 of testNormalScanTimeoutOnNextAsync failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '8336541563169538088' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T17:38:33,273 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNextAsync Thread=222 (was 222), OpenFileDescriptor=446 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=122 (was 116) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10023 (was 10033) 2024-11-27T17:38:33,282 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=122, ProcessCount=11, AvailableMemoryMB=10023 2024-11-27T17:38:33,285 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T17:38:33,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext 2024-11-27T17:38:33,288 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T17:38:33,289 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:38:33,289 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnNext" procId is: 22 2024-11-27T17:38:33,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-27T17:38:33,291 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T17:38:33,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741847_1023 (size=411) 2024-11-27T17:38:33,706 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 528929fce6443603ca9db935c1532830, NAME => 'testNormalScanTimeoutOnNext,,1732729113284.528929fce6443603ca9db935c1532830.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f 2024-11-27T17:38:33,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741848_1024 (size=62) 2024-11-27T17:38:33,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 78, skipping over this response of 82 bytes 2024-11-27T17:38:33,876 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T17:38:33,903 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync 2024-11-27T17:38:33,903 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNextAsync Metrics about Tables on a single HBase RegionServer 2024-11-27T17:38:33,904 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNextAsync' 2024-11-27T17:38:34,120 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732729113284.528929fce6443603ca9db935c1532830.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T17:38:34,120 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 528929fce6443603ca9db935c1532830, disabling compactions & flushes 2024-11-27T17:38:34,121 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732729113284.528929fce6443603ca9db935c1532830. 2024-11-27T17:38:34,121 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732729113284.528929fce6443603ca9db935c1532830. 2024-11-27T17:38:34,121 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732729113284.528929fce6443603ca9db935c1532830. after waiting 0 ms 2024-11-27T17:38:34,121 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732729113284.528929fce6443603ca9db935c1532830. 2024-11-27T17:38:34,121 INFO [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732729113284.528929fce6443603ca9db935c1532830. 2024-11-27T17:38:34,122 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 528929fce6443603ca9db935c1532830: Waiting for close lock at 1732729114120Disabling compacts and flushes for region at 1732729114120Disabling writes for close at 1732729114121 (+1 ms)Writing region close event to WAL at 1732729114121Closed at 1732729114121 2024-11-27T17:38:34,125 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T17:38:34,126 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnNext,,1732729113284.528929fce6443603ca9db935c1532830.","families":{"info":[{"qualifier":"regioninfo","vlen":61,"tag":[],"timestamp":"1732729114125"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732729114125"}]},"ts":"1732729114125"} 2024-11-27T17:38:34,130 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T17:38:34,132 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T17:38:34,132 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732729114132"}]},"ts":"1732729114132"} 2024-11-27T17:38:34,136 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-27T17:38:34,137 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=528929fce6443603ca9db935c1532830, ASSIGN}] 2024-11-27T17:38:34,138 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=528929fce6443603ca9db935c1532830, ASSIGN 2024-11-27T17:38:34,139 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=528929fce6443603ca9db935c1532830, ASSIGN; state=OFFLINE, location=8f0673442175,43899,1732729014292; forceNewPlan=false, retain=false 2024-11-27T17:38:34,290 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=528929fce6443603ca9db935c1532830, regionState=OPENING, regionLocation=8f0673442175,43899,1732729014292 2024-11-27T17:38:34,297 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=528929fce6443603ca9db935c1532830, ASSIGN because future has completed 2024-11-27T17:38:34,298 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 528929fce6443603ca9db935c1532830, server=8f0673442175,43899,1732729014292}] 2024-11-27T17:38:34,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-27T17:38:34,464 INFO [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnNext,,1732729113284.528929fce6443603ca9db935c1532830. 2024-11-27T17:38:34,464 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7752): Opening region: {ENCODED => 528929fce6443603ca9db935c1532830, NAME => 'testNormalScanTimeoutOnNext,,1732729113284.528929fce6443603ca9db935c1532830.', STARTKEY => '', ENDKEY => ''} 2024-11-27T17:38:34,465 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnNext 528929fce6443603ca9db935c1532830 2024-11-27T17:38:34,466 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnNext,,1732729113284.528929fce6443603ca9db935c1532830.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T17:38:34,466 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7794): checking encryption for 528929fce6443603ca9db935c1532830 2024-11-27T17:38:34,466 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7797): checking classloading for 528929fce6443603ca9db935c1532830 2024-11-27T17:38:34,469 INFO [StoreOpener-528929fce6443603ca9db935c1532830-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 528929fce6443603ca9db935c1532830 2024-11-27T17:38:34,471 INFO [StoreOpener-528929fce6443603ca9db935c1532830-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 528929fce6443603ca9db935c1532830 columnFamilyName testFamily 2024-11-27T17:38:34,471 DEBUG [StoreOpener-528929fce6443603ca9db935c1532830-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:38:34,472 INFO [StoreOpener-528929fce6443603ca9db935c1532830-1 {}] regionserver.HStore(327): Store=528929fce6443603ca9db935c1532830/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T17:38:34,472 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1038): replaying wal for 528929fce6443603ca9db935c1532830 2024-11-27T17:38:34,473 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnNext/528929fce6443603ca9db935c1532830 2024-11-27T17:38:34,473 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnNext/528929fce6443603ca9db935c1532830 2024-11-27T17:38:34,474 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1048): stopping wal replay for 528929fce6443603ca9db935c1532830 2024-11-27T17:38:34,474 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1060): Cleaning up temporary data for 528929fce6443603ca9db935c1532830 2024-11-27T17:38:34,476 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): writing seq id for 528929fce6443603ca9db935c1532830 2024-11-27T17:38:34,479 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnNext/528929fce6443603ca9db935c1532830/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T17:38:34,480 INFO [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1114): Opened 528929fce6443603ca9db935c1532830; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70323453, jitterRate=0.04790110886096954}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T17:38:34,480 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 528929fce6443603ca9db935c1532830 2024-11-27T17:38:34,481 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1006): Region open journal for 528929fce6443603ca9db935c1532830: Running coprocessor pre-open hook at 1732729114466Writing region info on filesystem at 1732729114466Initializing all the Stores at 1732729114468 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732729114469 (+1 ms)Cleaning up temporary data from old regions at 1732729114474 (+5 ms)Running coprocessor post-open hooks at 1732729114480 (+6 ms)Region opened successfully at 1732729114481 (+1 ms) 2024-11-27T17:38:34,482 INFO [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnNext,,1732729113284.528929fce6443603ca9db935c1532830., pid=24, masterSystemTime=1732729114452 2024-11-27T17:38:34,485 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnNext,,1732729113284.528929fce6443603ca9db935c1532830. 2024-11-27T17:38:34,485 INFO [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnNext,,1732729113284.528929fce6443603ca9db935c1532830. 2024-11-27T17:38:34,486 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=528929fce6443603ca9db935c1532830, regionState=OPEN, openSeqNum=2, regionLocation=8f0673442175,43899,1732729014292 2024-11-27T17:38:34,489 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 528929fce6443603ca9db935c1532830, server=8f0673442175,43899,1732729014292 because future has completed 2024-11-27T17:38:34,493 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-27T17:38:34,493 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 528929fce6443603ca9db935c1532830, server=8f0673442175,43899,1732729014292 in 193 msec 2024-11-27T17:38:34,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-27T17:38:34,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnNext, region=528929fce6443603ca9db935c1532830, ASSIGN in 357 msec 2024-11-27T17:38:34,497 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T17:38:34,497 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732729114497"}]},"ts":"1732729114497"} 2024-11-27T17:38:34,500 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-27T17:38:34,502 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T17:38:34,504 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnNext in 1.2170 sec 2024-11-27T17:38:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-11-27T17:38:36,324 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnNext completed 2024-11-27T17:38:36,324 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-27T17:38:36,324 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T17:38:36,335 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-27T17:38:36,335 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T17:38:36,335 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnNext assigned. 2024-11-27T17:38:36,339 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnNext,,1732729113284.528929fce6443603ca9db935c1532830., hostname=8f0673442175,43899,1732729014292, seqNum=2] 2024-11-27T17:38:36,349 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-27T17:38:36,349 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnNext,, stopping at row=testNormalScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-27T17:38:36,352 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-27T17:38:36,354 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-27T17:38:36,355 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-27T17:38:36,356 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43899 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 3500 2024-11-27T17:38:38,667 INFO [master/8f0673442175:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-27T17:38:38,667 INFO [master/8f0673442175:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-27T17:38:39,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43899 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '8336541563169538092' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-27T17:38:39,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43899 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:42734 deadline: 1732729120372, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '8336541563169538092' does not exist 2024-11-27T17:38:39,375 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 8f0673442175,43899,1732729014292 for closing scanner id = 8336541563169538092 for 528929fce6443603ca9db935c1532830 of testNormalScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '8336541563169538092' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T17:38:39,380 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnNext Thread=222 (was 222), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=112 (was 122), ProcessCount=11 (was 11), AvailableMemoryMB=10028 (was 10023) - AvailableMemoryMB LEAK? - 2024-11-27T17:38:39,388 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=112, ProcessCount=11, AvailableMemoryMB=10028 2024-11-27T17:38:39,391 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46243 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T17:38:39,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46243 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync 2024-11-27T17:38:39,395 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T17:38:39,395 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:38:39,396 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46243 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnOpenScannerAsync" procId is: 25 2024-11-27T17:38:39,397 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T17:38:39,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T17:38:39,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741849_1025 (size=419) 2024-11-27T17:38:39,812 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8ebfd4c85458b86adf7a32ac08f189f7, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732729119390.8ebfd4c85458b86adf7a32ac08f189f7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f 2024-11-27T17:38:39,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741850_1026 (size=70) 2024-11-27T17:38:39,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 88, skipping over this response of 82 bytes 2024-11-27T17:38:39,968 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T17:38:40,002 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnNext' 2024-11-27T17:38:40,225 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732729119390.8ebfd4c85458b86adf7a32ac08f189f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T17:38:40,225 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 8ebfd4c85458b86adf7a32ac08f189f7, disabling compactions & flushes 2024-11-27T17:38:40,225 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732729119390.8ebfd4c85458b86adf7a32ac08f189f7. 2024-11-27T17:38:40,226 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732729119390.8ebfd4c85458b86adf7a32ac08f189f7. 2024-11-27T17:38:40,226 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732729119390.8ebfd4c85458b86adf7a32ac08f189f7. after waiting 0 ms 2024-11-27T17:38:40,226 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732729119390.8ebfd4c85458b86adf7a32ac08f189f7. 2024-11-27T17:38:40,226 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732729119390.8ebfd4c85458b86adf7a32ac08f189f7. 2024-11-27T17:38:40,226 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8ebfd4c85458b86adf7a32ac08f189f7: Waiting for close lock at 1732729120225Disabling compacts and flushes for region at 1732729120225Disabling writes for close at 1732729120226 (+1 ms)Writing region close event to WAL at 1732729120226Closed at 1732729120226 2024-11-27T17:38:40,229 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T17:38:40,230 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732729119390.8ebfd4c85458b86adf7a32ac08f189f7.","families":{"info":[{"qualifier":"regioninfo","vlen":69,"tag":[],"timestamp":"1732729120230"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732729120230"}]},"ts":"1732729120230"} 2024-11-27T17:38:40,235 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T17:38:40,237 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T17:38:40,238 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732729120237"}]},"ts":"1732729120237"} 2024-11-27T17:38:40,241 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-27T17:38:40,241 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=8ebfd4c85458b86adf7a32ac08f189f7, ASSIGN}] 2024-11-27T17:38:40,243 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=8ebfd4c85458b86adf7a32ac08f189f7, ASSIGN 2024-11-27T17:38:40,245 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=8ebfd4c85458b86adf7a32ac08f189f7, ASSIGN; state=OFFLINE, location=8f0673442175,43899,1732729014292; forceNewPlan=false, retain=false 2024-11-27T17:38:40,397 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=8ebfd4c85458b86adf7a32ac08f189f7, regionState=OPENING, regionLocation=8f0673442175,43899,1732729014292 2024-11-27T17:38:40,405 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=8ebfd4c85458b86adf7a32ac08f189f7, ASSIGN because future has completed 2024-11-27T17:38:40,406 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ebfd4c85458b86adf7a32ac08f189f7, server=8f0673442175,43899,1732729014292}] 2024-11-27T17:38:40,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T17:38:40,570 INFO [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732729119390.8ebfd4c85458b86adf7a32ac08f189f7. 2024-11-27T17:38:40,570 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7752): Opening region: {ENCODED => 8ebfd4c85458b86adf7a32ac08f189f7, NAME => 'hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732729119390.8ebfd4c85458b86adf7a32ac08f189f7.', STARTKEY => '', ENDKEY => ''} 2024-11-27T17:38:40,571 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnOpenScannerAsync 8ebfd4c85458b86adf7a32ac08f189f7 2024-11-27T17:38:40,571 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732729119390.8ebfd4c85458b86adf7a32ac08f189f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T17:38:40,571 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7794): checking encryption for 8ebfd4c85458b86adf7a32ac08f189f7 2024-11-27T17:38:40,571 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(7797): checking classloading for 8ebfd4c85458b86adf7a32ac08f189f7 2024-11-27T17:38:40,573 INFO [StoreOpener-8ebfd4c85458b86adf7a32ac08f189f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 8ebfd4c85458b86adf7a32ac08f189f7 2024-11-27T17:38:40,576 INFO [StoreOpener-8ebfd4c85458b86adf7a32ac08f189f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ebfd4c85458b86adf7a32ac08f189f7 columnFamilyName testFamily 2024-11-27T17:38:40,576 DEBUG [StoreOpener-8ebfd4c85458b86adf7a32ac08f189f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:38:40,576 INFO [StoreOpener-8ebfd4c85458b86adf7a32ac08f189f7-1 {}] regionserver.HStore(327): Store=8ebfd4c85458b86adf7a32ac08f189f7/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T17:38:40,577 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1038): replaying wal for 8ebfd4c85458b86adf7a32ac08f189f7 2024-11-27T17:38:40,578 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/8ebfd4c85458b86adf7a32ac08f189f7 2024-11-27T17:38:40,578 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/8ebfd4c85458b86adf7a32ac08f189f7 2024-11-27T17:38:40,579 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1048): stopping wal replay for 8ebfd4c85458b86adf7a32ac08f189f7 2024-11-27T17:38:40,579 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1060): Cleaning up temporary data for 8ebfd4c85458b86adf7a32ac08f189f7 2024-11-27T17:38:40,581 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1093): writing seq id for 8ebfd4c85458b86adf7a32ac08f189f7 2024-11-27T17:38:40,584 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/8ebfd4c85458b86adf7a32ac08f189f7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T17:38:40,585 INFO [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1114): Opened 8ebfd4c85458b86adf7a32ac08f189f7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68868163, jitterRate=0.026215597987174988}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T17:38:40,585 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8ebfd4c85458b86adf7a32ac08f189f7 2024-11-27T17:38:40,585 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegion(1006): Region open journal for 8ebfd4c85458b86adf7a32ac08f189f7: Running coprocessor pre-open hook at 1732729120571Writing region info on filesystem at 1732729120571Initializing all the Stores at 1732729120573 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732729120573Cleaning up temporary data from old regions at 1732729120579 (+6 ms)Running coprocessor post-open hooks at 1732729120585 (+6 ms)Region opened successfully at 1732729120585 2024-11-27T17:38:40,587 INFO [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732729119390.8ebfd4c85458b86adf7a32ac08f189f7., pid=27, masterSystemTime=1732729120559 2024-11-27T17:38:40,589 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732729119390.8ebfd4c85458b86adf7a32ac08f189f7. 2024-11-27T17:38:40,589 INFO [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=27}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732729119390.8ebfd4c85458b86adf7a32ac08f189f7. 2024-11-27T17:38:40,590 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=8ebfd4c85458b86adf7a32ac08f189f7, regionState=OPEN, openSeqNum=2, regionLocation=8f0673442175,43899,1732729014292 2024-11-27T17:38:40,593 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ebfd4c85458b86adf7a32ac08f189f7, server=8f0673442175,43899,1732729014292 because future has completed 2024-11-27T17:38:40,596 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-27T17:38:40,596 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 8ebfd4c85458b86adf7a32ac08f189f7, server=8f0673442175,43899,1732729014292 in 188 msec 2024-11-27T17:38:40,599 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-27T17:38:40,599 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync, region=8ebfd4c85458b86adf7a32ac08f189f7, ASSIGN in 355 msec 2024-11-27T17:38:40,600 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T17:38:40,601 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732729120600"}]},"ts":"1732729120600"} 2024-11-27T17:38:40,603 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-27T17:38:40,605 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=25, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T17:38:40,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnOpenScannerAsync in 1.2140 sec 2024-11-27T17:38:42,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T17:38:42,434 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnOpenScannerAsync completed 2024-11-27T17:38:42,434 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-27T17:38:42,435 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T17:38:42,448 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-27T17:38:42,448 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T17:38:42,448 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnOpenScannerAsync assigned. 2024-11-27T17:38:42,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732729119390.8ebfd4c85458b86adf7a32ac08f189f7., hostname=8f0673442175,43899,1732729014292, seqNum=2] 2024-11-27T17:38:42,461 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-27T17:38:42,461 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnOpenScannerAsync,, stopping at row=hbase:testMetaScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-27T17:38:42,464 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-27T17:38:42,466 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43899 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 6500 2024-11-27T17:38:43,676 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext 2024-11-27T17:38:43,676 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-27T17:38:43,678 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync 2024-11-27T17:38:43,679 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-27T17:38:46,075 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T17:38:46,106 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnOpenScannerAsync' 2024-11-27T17:38:48,475 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732729119390.8ebfd4c85458b86adf7a32ac08f189f7., hostname=8f0673442175,43899,1732729014292, seqNum=2 , the old value is region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732729119390.8ebfd4c85458b86adf7a32ac08f189f7., hostname=8f0673442175,43899,1732729014292, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=8f0673442175:43899 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6010ms, rpcTimeout=5999ms 2024-11-27T17:38:48,476 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732729119390.8ebfd4c85458b86adf7a32ac08f189f7., hostname=8f0673442175,43899,1732729014292, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=96,methodName=Scan], waitTime=6010ms, rpcTimeout=5999ms 2024-11-27T17:38:48,476 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732729119390.8ebfd4c85458b86adf7a32ac08f189f7., hostname=8f0673442175,43899,1732729014292, seqNum=2 from cache 2024-11-27T17:38:48,492 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnOpenScannerAsync Thread=222 (was 222), OpenFileDescriptor=445 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=110 (was 112), ProcessCount=11 (was 11), AvailableMemoryMB=10020 (was 10028) 2024-11-27T17:38:48,502 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=222, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=110, ProcessCount=11, AvailableMemoryMB=10019 2024-11-27T17:38:48,504 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46243 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T17:38:48,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46243 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext 2024-11-27T17:38:48,507 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T17:38:48,507 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:38:48,507 INFO [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46243 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "hbase" qualifier: "testMetaScanTimeoutOnNext" procId is: 28 2024-11-27T17:38:48,509 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T17:38:48,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-27T17:38:48,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741851_1027 (size=407) 2024-11-27T17:38:48,925 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0105071c31a7ce20d864bf590e719c24, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732729128504.0105071c31a7ce20d864bf590e719c24.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:testMetaScanTimeoutOnNext', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f 2024-11-27T17:38:48,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741852_1028 (size=58) 2024-11-27T17:38:48,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 96, skipping over this response of 88 bytes 2024-11-27T17:38:49,339 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732729128504.0105071c31a7ce20d864bf590e719c24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T17:38:49,340 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1722): Closing 0105071c31a7ce20d864bf590e719c24, disabling compactions & flushes 2024-11-27T17:38:49,340 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732729128504.0105071c31a7ce20d864bf590e719c24. 2024-11-27T17:38:49,340 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732729128504.0105071c31a7ce20d864bf590e719c24. 2024-11-27T17:38:49,340 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732729128504.0105071c31a7ce20d864bf590e719c24. after waiting 0 ms 2024-11-27T17:38:49,341 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732729128504.0105071c31a7ce20d864bf590e719c24. 2024-11-27T17:38:49,341 INFO [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732729128504.0105071c31a7ce20d864bf590e719c24. 2024-11-27T17:38:49,341 DEBUG [RegionOpenAndInit-hbase:testMetaScanTimeoutOnNext-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0105071c31a7ce20d864bf590e719c24: Waiting for close lock at 1732729129340Disabling compacts and flushes for region at 1732729129340Disabling writes for close at 1732729129340Writing region close event to WAL at 1732729129341 (+1 ms)Closed at 1732729129341 2024-11-27T17:38:49,346 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T17:38:49,346 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:testMetaScanTimeoutOnNext,,1732729128504.0105071c31a7ce20d864bf590e719c24.","families":{"info":[{"qualifier":"regioninfo","vlen":57,"tag":[],"timestamp":"1732729129346"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732729129346"}]},"ts":"1732729129346"} 2024-11-27T17:38:49,350 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T17:38:49,352 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T17:38:49,352 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732729129352"}]},"ts":"1732729129352"} 2024-11-27T17:38:49,354 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLING in hbase:meta 2024-11-27T17:38:49,354 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=0105071c31a7ce20d864bf590e719c24, ASSIGN}] 2024-11-27T17:38:49,356 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=0105071c31a7ce20d864bf590e719c24, ASSIGN 2024-11-27T17:38:49,356 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=0105071c31a7ce20d864bf590e719c24, ASSIGN; state=OFFLINE, location=8f0673442175,43899,1732729014292; forceNewPlan=false, retain=false 2024-11-27T17:38:49,508 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=0105071c31a7ce20d864bf590e719c24, regionState=OPENING, regionLocation=8f0673442175,43899,1732729014292 2024-11-27T17:38:49,516 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=0105071c31a7ce20d864bf590e719c24, ASSIGN because future has completed 2024-11-27T17:38:49,516 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0105071c31a7ce20d864bf590e719c24, server=8f0673442175,43899,1732729014292}] 2024-11-27T17:38:49,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-27T17:38:49,679 INFO [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(132): Open hbase:testMetaScanTimeoutOnNext,,1732729128504.0105071c31a7ce20d864bf590e719c24. 2024-11-27T17:38:49,679 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 0105071c31a7ce20d864bf590e719c24, NAME => 'hbase:testMetaScanTimeoutOnNext,,1732729128504.0105071c31a7ce20d864bf590e719c24.', STARTKEY => '', ENDKEY => ''} 2024-11-27T17:38:49,680 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testMetaScanTimeoutOnNext 0105071c31a7ce20d864bf590e719c24 2024-11-27T17:38:49,680 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(898): Instantiated hbase:testMetaScanTimeoutOnNext,,1732729128504.0105071c31a7ce20d864bf590e719c24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T17:38:49,680 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 0105071c31a7ce20d864bf590e719c24 2024-11-27T17:38:49,680 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 0105071c31a7ce20d864bf590e719c24 2024-11-27T17:38:49,683 INFO [StoreOpener-0105071c31a7ce20d864bf590e719c24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 0105071c31a7ce20d864bf590e719c24 2024-11-27T17:38:49,685 INFO [StoreOpener-0105071c31a7ce20d864bf590e719c24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0105071c31a7ce20d864bf590e719c24 columnFamilyName testFamily 2024-11-27T17:38:49,686 DEBUG [StoreOpener-0105071c31a7ce20d864bf590e719c24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:38:49,686 INFO [StoreOpener-0105071c31a7ce20d864bf590e719c24-1 {}] regionserver.HStore(327): Store=0105071c31a7ce20d864bf590e719c24/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T17:38:49,687 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 0105071c31a7ce20d864bf590e719c24 2024-11-27T17:38:49,688 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnNext/0105071c31a7ce20d864bf590e719c24 2024-11-27T17:38:49,688 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnNext/0105071c31a7ce20d864bf590e719c24 2024-11-27T17:38:49,689 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 0105071c31a7ce20d864bf590e719c24 2024-11-27T17:38:49,689 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 0105071c31a7ce20d864bf590e719c24 2024-11-27T17:38:49,692 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 0105071c31a7ce20d864bf590e719c24 2024-11-27T17:38:49,697 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnNext/0105071c31a7ce20d864bf590e719c24/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T17:38:49,698 INFO [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1114): Opened 0105071c31a7ce20d864bf590e719c24; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59566248, jitterRate=-0.11239373683929443}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T17:38:49,698 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0105071c31a7ce20d864bf590e719c24 2024-11-27T17:38:49,699 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 0105071c31a7ce20d864bf590e719c24: Running coprocessor pre-open hook at 1732729129681Writing region info on filesystem at 1732729129681Initializing all the Stores at 1732729129683 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732729129683Cleaning up temporary data from old regions at 1732729129689 (+6 ms)Running coprocessor post-open hooks at 1732729129698 (+9 ms)Region opened successfully at 1732729129699 (+1 ms) 2024-11-27T17:38:49,700 INFO [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:testMetaScanTimeoutOnNext,,1732729128504.0105071c31a7ce20d864bf590e719c24., pid=30, masterSystemTime=1732729129671 2024-11-27T17:38:49,703 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:testMetaScanTimeoutOnNext,,1732729128504.0105071c31a7ce20d864bf590e719c24. 2024-11-27T17:38:49,703 INFO [RS_OPEN_PRIORITY_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=30}] handler.AssignRegionHandler(153): Opened hbase:testMetaScanTimeoutOnNext,,1732729128504.0105071c31a7ce20d864bf590e719c24. 2024-11-27T17:38:49,704 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=0105071c31a7ce20d864bf590e719c24, regionState=OPEN, openSeqNum=2, regionLocation=8f0673442175,43899,1732729014292 2024-11-27T17:38:49,707 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0105071c31a7ce20d864bf590e719c24, server=8f0673442175,43899,1732729014292 because future has completed 2024-11-27T17:38:49,712 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-27T17:38:49,713 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 0105071c31a7ce20d864bf590e719c24, server=8f0673442175,43899,1732729014292 in 192 msec 2024-11-27T17:38:49,714 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-27T17:38:49,715 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:testMetaScanTimeoutOnNext, region=0105071c31a7ce20d864bf590e719c24, ASSIGN in 358 msec 2024-11-27T17:38:49,715 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T17:38:49,716 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:testMetaScanTimeoutOnNext","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732729129716"}]},"ts":"1732729129716"} 2024-11-27T17:38:49,718 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:testMetaScanTimeoutOnNext, state=ENABLED in hbase:meta 2024-11-27T17:38:49,719 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T17:38:49,722 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:testMetaScanTimeoutOnNext in 1.2150 sec 2024-11-27T17:38:51,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-27T17:38:51,534 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: hbase:testMetaScanTimeoutOnNext completed 2024-11-27T17:38:51,534 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table hbase:testMetaScanTimeoutOnNext get assigned. Timeout = 60000ms 2024-11-27T17:38:51,535 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T17:38:51,547 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table hbase:testMetaScanTimeoutOnNext assigned to meta. Checking AM states. 2024-11-27T17:38:51,547 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T17:38:51,547 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table hbase:testMetaScanTimeoutOnNext assigned. 2024-11-27T17:38:51,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:testMetaScanTimeoutOnNext', row='row-0', locateType=CURRENT is [region=hbase:testMetaScanTimeoutOnNext,,1732729128504.0105071c31a7ce20d864bf590e719c24., hostname=8f0673442175,43899,1732729014292, seqNum=2] 2024-11-27T17:38:51,561 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-27T17:38:51,561 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=hbase:testMetaScanTimeoutOnNext,, stopping at row=hbase:testMetaScanTimeoutOnNext ,, for max=2147483647 with caching=100 2024-11-27T17:38:51,565 INFO [Time-limited test {}] client.TestClientScannerTimeouts(313): Opening scanner, expecting no timeouts from first next() call from openScanner response 2024-11-27T17:38:51,567 INFO [Time-limited test {}] client.TestClientScannerTimeouts(319): Making first next() RPC, expecting no timeout for seqNo 0 2024-11-27T17:38:51,568 INFO [Time-limited test {}] client.TestClientScannerTimeouts(323): Making second next() RPC, expecting timeout 2024-11-27T17:38:51,569 INFO [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43899 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(445): SLEEPING 6500 2024-11-27T17:38:52,109 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:testMetaScanTimeoutOnNext' 2024-11-27T17:38:52,473 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T17:38:53,676 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext 2024-11-27T17:38:53,676 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_testMetaScanTimeoutOnNext Metrics about Tables on a single HBase RegionServer 2024-11-27T17:38:55,185 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T17:38:57,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43899 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.LeaseException: lease '8336541563169538100' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) ~[classes/:?] at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) ~[test-classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:?] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:?] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:?] 2024-11-27T17:38:57,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=43899 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Scan size: 40 connection: 172.17.0.3:42734 deadline: 1732729143577, exception=org.apache.hadoop.hbase.regionserver.LeaseException: lease '8336541563169538100' does not exist 2024-11-27T17:38:57,585 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncScanSingleRegionRpcRetryingCaller(378): Call to 8f0673442175,43899,1732729014292 for closing scanner id = 8336541563169538100 for 0105071c31a7ce20d864bf590e719c24 of hbase:testMetaScanTimeoutOnNext failed, ignore, probably already closed org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.regionserver.LeaseException: lease '8336541563169538100' does not exist at org.apache.hadoop.hbase.regionserver.LeaseManager.removeLease(LeaseManager.java:196) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:3597) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout.scan(TestClientScannerTimeouts.java:427) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43508) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T17:38:57,593 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testMetaScanTimeoutOnNext Thread=221 (was 222), OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=101 (was 110), ProcessCount=11 (was 11), AvailableMemoryMB=9992 (was 10019) 2024-11-27T17:38:57,599 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=221, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=101, ProcessCount=11, AvailableMemoryMB=9992 2024-11-27T17:38:57,601 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T17:38:57,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync 2024-11-27T17:38:57,604 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T17:38:57,604 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:38:57,604 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testNormalScanTimeoutOnOpenScannerAsync" procId is: 31 2024-11-27T17:38:57,605 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T17:38:57,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-27T17:38:57,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741853_1029 (size=423) 2024-11-27T17:38:58,022 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6de5dbf4f8dbf9b8abcc9963e93407d3, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testNormalScanTimeoutOnOpenScannerAsync', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f 2024-11-27T17:38:58,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741854_1030 (size=74) 2024-11-27T17:38:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 105, skipping over this response of 82 bytes 2024-11-27T17:38:58,433 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T17:38:58,433 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1722): Closing 6de5dbf4f8dbf9b8abcc9963e93407d3, disabling compactions & flushes 2024-11-27T17:38:58,434 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3. 2024-11-27T17:38:58,434 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3. 2024-11-27T17:38:58,434 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3. after waiting 0 ms 2024-11-27T17:38:58,434 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3. 2024-11-27T17:38:58,434 INFO [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3. 2024-11-27T17:38:58,434 DEBUG [RegionOpenAndInit-testNormalScanTimeoutOnOpenScannerAsync-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6de5dbf4f8dbf9b8abcc9963e93407d3: Waiting for close lock at 1732729138433Disabling compacts and flushes for region at 1732729138433Disabling writes for close at 1732729138434 (+1 ms)Writing region close event to WAL at 1732729138434Closed at 1732729138434 2024-11-27T17:38:58,438 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T17:38:58,439 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3.","families":{"info":[{"qualifier":"regioninfo","vlen":73,"tag":[],"timestamp":"1732729138438"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732729138438"}]},"ts":"1732729138438"} 2024-11-27T17:38:58,443 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T17:38:58,444 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T17:38:58,445 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732729138444"}]},"ts":"1732729138444"} 2024-11-27T17:38:58,448 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLING in hbase:meta 2024-11-27T17:38:58,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=6de5dbf4f8dbf9b8abcc9963e93407d3, ASSIGN}] 2024-11-27T17:38:58,450 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=6de5dbf4f8dbf9b8abcc9963e93407d3, ASSIGN 2024-11-27T17:38:58,451 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=6de5dbf4f8dbf9b8abcc9963e93407d3, ASSIGN; state=OFFLINE, location=8f0673442175,43899,1732729014292; forceNewPlan=false, retain=false 2024-11-27T17:38:58,602 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=6de5dbf4f8dbf9b8abcc9963e93407d3, regionState=OPENING, regionLocation=8f0673442175,43899,1732729014292 2024-11-27T17:38:58,608 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=6de5dbf4f8dbf9b8abcc9963e93407d3, ASSIGN because future has completed 2024-11-27T17:38:58,609 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6de5dbf4f8dbf9b8abcc9963e93407d3, server=8f0673442175,43899,1732729014292}] 2024-11-27T17:38:58,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-27T17:38:58,775 INFO [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3. 2024-11-27T17:38:58,775 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 6de5dbf4f8dbf9b8abcc9963e93407d3, NAME => 'testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3.', STARTKEY => '', ENDKEY => ''} 2024-11-27T17:38:58,775 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testNormalScanTimeoutOnOpenScannerAsync 6de5dbf4f8dbf9b8abcc9963e93407d3 2024-11-27T17:38:58,776 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T17:38:58,776 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 6de5dbf4f8dbf9b8abcc9963e93407d3 2024-11-27T17:38:58,776 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 6de5dbf4f8dbf9b8abcc9963e93407d3 2024-11-27T17:38:58,778 INFO [StoreOpener-6de5dbf4f8dbf9b8abcc9963e93407d3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family testFamily of region 6de5dbf4f8dbf9b8abcc9963e93407d3 2024-11-27T17:38:58,780 INFO [StoreOpener-6de5dbf4f8dbf9b8abcc9963e93407d3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6de5dbf4f8dbf9b8abcc9963e93407d3 columnFamilyName testFamily 2024-11-27T17:38:58,780 DEBUG [StoreOpener-6de5dbf4f8dbf9b8abcc9963e93407d3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T17:38:58,781 INFO [StoreOpener-6de5dbf4f8dbf9b8abcc9963e93407d3-1 {}] regionserver.HStore(327): Store=6de5dbf4f8dbf9b8abcc9963e93407d3/testFamily, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T17:38:58,781 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 6de5dbf4f8dbf9b8abcc9963e93407d3 2024-11-27T17:38:58,782 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnOpenScannerAsync/6de5dbf4f8dbf9b8abcc9963e93407d3 2024-11-27T17:38:58,783 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnOpenScannerAsync/6de5dbf4f8dbf9b8abcc9963e93407d3 2024-11-27T17:38:58,784 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 6de5dbf4f8dbf9b8abcc9963e93407d3 2024-11-27T17:38:58,784 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 6de5dbf4f8dbf9b8abcc9963e93407d3 2024-11-27T17:38:58,786 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 6de5dbf4f8dbf9b8abcc9963e93407d3 2024-11-27T17:38:58,790 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnOpenScannerAsync/6de5dbf4f8dbf9b8abcc9963e93407d3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T17:38:58,790 INFO [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 6de5dbf4f8dbf9b8abcc9963e93407d3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69701339, jitterRate=0.0386308878660202}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T17:38:58,791 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6de5dbf4f8dbf9b8abcc9963e93407d3 2024-11-27T17:38:58,792 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 6de5dbf4f8dbf9b8abcc9963e93407d3: Running coprocessor pre-open hook at 1732729138776Writing region info on filesystem at 1732729138776Initializing all the Stores at 1732729138778 (+2 ms)Instantiating store for column family {NAME => 'testFamily', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732729138778Cleaning up temporary data from old regions at 1732729138784 (+6 ms)Running coprocessor post-open hooks at 1732729138791 (+7 ms)Region opened successfully at 1732729138792 (+1 ms) 2024-11-27T17:38:58,793 INFO [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3., pid=33, masterSystemTime=1732729138763 2024-11-27T17:38:58,796 DEBUG [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3. 2024-11-27T17:38:58,796 INFO [RS_OPEN_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3. 2024-11-27T17:38:58,797 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=32 updating hbase:meta row=6de5dbf4f8dbf9b8abcc9963e93407d3, regionState=OPEN, openSeqNum=2, regionLocation=8f0673442175,43899,1732729014292 2024-11-27T17:38:58,799 DEBUG [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6de5dbf4f8dbf9b8abcc9963e93407d3, server=8f0673442175,43899,1732729014292 because future has completed 2024-11-27T17:38:58,803 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-27T17:38:58,803 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; OpenRegionProcedure 6de5dbf4f8dbf9b8abcc9963e93407d3, server=8f0673442175,43899,1732729014292 in 191 msec 2024-11-27T17:38:58,805 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-27T17:38:58,805 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testNormalScanTimeoutOnOpenScannerAsync, region=6de5dbf4f8dbf9b8abcc9963e93407d3, ASSIGN in 355 msec 2024-11-27T17:38:58,806 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T17:38:58,806 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testNormalScanTimeoutOnOpenScannerAsync","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732729138806"}]},"ts":"1732729138806"} 2024-11-27T17:38:58,808 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testNormalScanTimeoutOnOpenScannerAsync, state=ENABLED in hbase:meta 2024-11-27T17:38:58,809 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T17:38:58,811 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; CreateTableProcedure table=testNormalScanTimeoutOnOpenScannerAsync in 1.2080 sec 2024-11-27T17:38:59,814 DEBUG [master/8f0673442175:0.Chore.1 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-27T17:39:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46243 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-27T17:39:00,643 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testNormalScanTimeoutOnOpenScannerAsync completed 2024-11-27T17:39:00,643 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testNormalScanTimeoutOnOpenScannerAsync get assigned. Timeout = 60000ms 2024-11-27T17:39:00,644 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T17:39:00,654 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned to meta. Checking AM states. 2024-11-27T17:39:00,654 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T17:39:00,654 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testNormalScanTimeoutOnOpenScannerAsync assigned. 2024-11-27T17:39:00,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='row-0', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3., hostname=8f0673442175,43899,1732729014292, seqNum=2] 2024-11-27T17:39:00,667 INFO [Time-limited test {}] client.TestClientScannerTimeouts(130): Wrote our four values 2024-11-27T17:39:00,667 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=testNormalScanTimeoutOnOpenScannerAsync,, stopping at row=testNormalScanTimeoutOnOpenScannerAsync ,, for max=2147483647 with caching=100 2024-11-27T17:39:00,671 INFO [Time-limited test {}] client.TestClientScannerTimeouts(338): Opening scanner, expecting timeout from first next() call from openScanner response 2024-11-27T17:39:00,673 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43899 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-27T17:39:01,676 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3., hostname=8f0673442175,43899,1732729014292, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3., hostname=8f0673442175,43899,1732729014292, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=8f0673442175:43899 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2024-11-27T17:39:01,676 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3., hostname=8f0673442175,43899,1732729014292, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=113,methodName=Scan], waitTime=1003ms, rpcTimeout=1000ms 2024-11-27T17:39:01,676 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3., hostname=8f0673442175,43899,1732729014292, seqNum=2 from cache 2024-11-27T17:39:02,175 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(450): Unknown callId: 113, skipping over this response of 88 bytes 2024-11-27T17:39:02,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testNormalScanTimeoutOnOpenScannerAsync', row='', locateType=CURRENT is [region=testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3., hostname=8f0673442175,43899,1732729014292, seqNum=2] 2024-11-27T17:39:02,702 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43899 {}] client.TestClientScannerTimeouts$RSRpcServicesWithScanTimeout(460): openScanner SLEEPING 1500 2024-11-27T17:39:03,675 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3., hostname=8f0673442175,43899,1732729014292, seqNum=2 , the old value is region=testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3., hostname=8f0673442175,43899,1732729014292, seqNum=2, error=org.apache.hadoop.hbase.ipc.CallTimeoutException: Call to address=8f0673442175:43899 failed on local exception: org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=974ms, rpcTimeout=971ms 2024-11-27T17:39:03,675 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3., hostname=8f0673442175,43899,1732729014292, seqNum=2 is org.apache.hadoop.hbase.ipc.CallTimeoutException: Call[id=115,methodName=Scan], waitTime=974ms, rpcTimeout=971ms 2024-11-27T17:39:03,676 DEBUG [RpcClient-timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3., hostname=8f0673442175,43899,1732729014292, seqNum=2 from cache 2024-11-27T17:39:03,676 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync 2024-11-27T17:39:03,676 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testNormalScanTimeoutOnOpenScannerAsync Metrics about Tables on a single HBase RegionServer 2024-11-27T17:39:03,689 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestClientScannerTimeouts#testNormalScanTimeoutOnOpenScannerAsync Thread=222 (was 221) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=93 (was 101), ProcessCount=11 (was 11), AvailableMemoryMB=10000 (was 9992) - AvailableMemoryMB LEAK? - 2024-11-27T17:39:03,690 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T17:39:03,690 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:109) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T17:39:03,696 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T17:39:03,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T17:39:03,697 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T17:39:03,697 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-27T17:39:03,697 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T17:39:03,697 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T17:39:03,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T17:39:03,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T17:39:03,697 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T17:39:03,698 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-27T17:39:03,698 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=177004388, stopped=false 2024-11-27T17:39:03,698 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=8f0673442175,46243,1732729013300 2024-11-27T17:39:03,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43899-0x1017d5e7cd30001, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T17:39:03,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46243-0x1017d5e7cd30000, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T17:39:03,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43899-0x1017d5e7cd30001, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T17:39:03,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46243-0x1017d5e7cd30000, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T17:39:03,766 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T17:39:03,767 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T17:39:03,767 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46243-0x1017d5e7cd30000, quorum=127.0.0.1:64480, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T17:39:03,767 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestClientScannerTimeouts.tearDownAfterClass(TestClientScannerTimeouts.java:111) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T17:39:03,768 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43899-0x1017d5e7cd30001, quorum=127.0.0.1:64480, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T17:39:03,768 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T17:39:03,769 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '8f0673442175,43899,1732729014292' ***** 2024-11-27T17:39:03,769 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T17:39:03,770 INFO [RS:0;8f0673442175:43899 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T17:39:03,770 INFO [RS:0;8f0673442175:43899 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T17:39:03,770 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T17:39:03,771 INFO [RS:0;8f0673442175:43899 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T17:39:03,771 INFO [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(3091): Received CLOSE for 34d7236d8a6e21ffac402f86622f6a1d 2024-11-27T17:39:03,772 INFO [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(3091): Received CLOSE for 528929fce6443603ca9db935c1532830 2024-11-27T17:39:03,772 INFO [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(3091): Received CLOSE for 6de5dbf4f8dbf9b8abcc9963e93407d3 2024-11-27T17:39:03,772 INFO [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(3091): Received CLOSE for a82ab4b705fac222998b7c9c55b9d4a6 2024-11-27T17:39:03,772 INFO [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(3091): Received CLOSE for 3fd0994b61fc0961c034fe9da87dbbfd 2024-11-27T17:39:03,772 INFO [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(3091): Received CLOSE for 87e3e592443f2e8ffc03e943933578f6 2024-11-27T17:39:03,773 INFO [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(3091): Received CLOSE for 0105071c31a7ce20d864bf590e719c24 2024-11-27T17:39:03,773 INFO [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(3091): Received CLOSE for a46374dfd125bdd1346773a9c685bf28 2024-11-27T17:39:03,773 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 34d7236d8a6e21ffac402f86622f6a1d, disabling compactions & flushes 2024-11-27T17:39:03,773 INFO [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(3091): Received CLOSE for 8ebfd4c85458b86adf7a32ac08f189f7 2024-11-27T17:39:03,773 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScanner,,1732729081763.34d7236d8a6e21ffac402f86622f6a1d. 2024-11-27T17:39:03,773 INFO [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(3091): Received CLOSE for f048b20286d5f531019fd9a36448940a 2024-11-27T17:39:03,773 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732729081763.34d7236d8a6e21ffac402f86622f6a1d. 2024-11-27T17:39:03,773 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScanner,,1732729081763.34d7236d8a6e21ffac402f86622f6a1d. after waiting 0 ms 2024-11-27T17:39:03,773 INFO [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(959): stopping server 8f0673442175,43899,1732729014292 2024-11-27T17:39:03,773 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScanner,,1732729081763.34d7236d8a6e21ffac402f86622f6a1d. 2024-11-27T17:39:03,773 INFO [RS:0;8f0673442175:43899 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T17:39:03,773 INFO [RS:0;8f0673442175:43899 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;8f0673442175:43899. 2024-11-27T17:39:03,773 DEBUG [RS:0;8f0673442175:43899 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T17:39:03,773 DEBUG [RS:0;8f0673442175:43899 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T17:39:03,773 INFO [RS:0;8f0673442175:43899 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T17:39:03,774 INFO [RS:0;8f0673442175:43899 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T17:39:03,774 INFO [RS:0;8f0673442175:43899 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T17:39:03,774 INFO [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-27T17:39:03,774 INFO [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1321): Waiting on 11 regions to close 2024-11-27T17:39:03,774 DEBUG [RS_CLOSE_META-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T17:39:03,774 INFO [RS_CLOSE_META-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T17:39:03,774 DEBUG [RS_CLOSE_META-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T17:39:03,774 DEBUG [RS_CLOSE_META-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T17:39:03,774 DEBUG [RS_CLOSE_META-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T17:39:03,774 DEBUG [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1351): Waiting on 0105071c31a7ce20d864bf590e719c24, 1588230740, 34d7236d8a6e21ffac402f86622f6a1d, 3fd0994b61fc0961c034fe9da87dbbfd, 528929fce6443603ca9db935c1532830, 6de5dbf4f8dbf9b8abcc9963e93407d3, 87e3e592443f2e8ffc03e943933578f6, 8ebfd4c85458b86adf7a32ac08f189f7, a46374dfd125bdd1346773a9c685bf28, a82ab4b705fac222998b7c9c55b9d4a6, f048b20286d5f531019fd9a36448940a 2024-11-27T17:39:03,777 INFO [RS_CLOSE_META-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=15.68 KB heapSize=25.68 KB 2024-11-27T17:39:03,777 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 34d7236d8a6e21ffac402f86622f6a1d 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-27T17:39:03,841 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnOpenScanner/34d7236d8a6e21ffac402f86622f6a1d/.tmp/testFamily/2277260fa62c4e71bb8aa5a99ac22005 is 61, key is row-0/testFamily:testQualifier/1732729084820/Put/seqid=0 2024-11-27T17:39:03,846 DEBUG [RS_CLOSE_META-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/meta/1588230740/.tmp/info/67022caf71aa4ad3bed7ed326a6a5767 is 209, key is testRetryOutOfOrderScannerNextExceptionAsync,,1732729078582.a46374dfd125bdd1346773a9c685bf28./info:regioninfo/1732729079840/Put/seqid=0 2024-11-27T17:39:03,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741855_1031 (size=5049) 2024-11-27T17:39:03,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741856_1032 (size=20564) 2024-11-27T17:39:03,975 DEBUG [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1351): Waiting on 0105071c31a7ce20d864bf590e719c24, 1588230740, 34d7236d8a6e21ffac402f86622f6a1d, 3fd0994b61fc0961c034fe9da87dbbfd, 528929fce6443603ca9db935c1532830, 6de5dbf4f8dbf9b8abcc9963e93407d3, 87e3e592443f2e8ffc03e943933578f6, 8ebfd4c85458b86adf7a32ac08f189f7, a46374dfd125bdd1346773a9c685bf28, a82ab4b705fac222998b7c9c55b9d4a6, f048b20286d5f531019fd9a36448940a 2024-11-27T17:39:04,175 DEBUG [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1351): Waiting on 0105071c31a7ce20d864bf590e719c24, 1588230740, 34d7236d8a6e21ffac402f86622f6a1d, 3fd0994b61fc0961c034fe9da87dbbfd, 528929fce6443603ca9db935c1532830, 6de5dbf4f8dbf9b8abcc9963e93407d3, 87e3e592443f2e8ffc03e943933578f6, 8ebfd4c85458b86adf7a32ac08f189f7, a46374dfd125bdd1346773a9c685bf28, a82ab4b705fac222998b7c9c55b9d4a6, f048b20286d5f531019fd9a36448940a 2024-11-27T17:39:04,203 WARN [RegionServerWithScanTimeout-EventLoopGroup-3-3 {}] util.NettyFutureUtils(65): IO operation failed org.apache.hbase.thirdparty.io.netty.channel.StacklessClosedChannelException: null at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AbstractUnsafe.write(Object, ChannelPromise)(Unknown Source) ~[hbase-shaded-netty-4.1.9.jar:?] 2024-11-27T17:39:04,257 INFO [RS_CLOSE_META-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.27 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/meta/1588230740/.tmp/info/67022caf71aa4ad3bed7ed326a6a5767 2024-11-27T17:39:04,257 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnOpenScanner/34d7236d8a6e21ffac402f86622f6a1d/.tmp/testFamily/2277260fa62c4e71bb8aa5a99ac22005 2024-11-27T17:39:04,277 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T17:39:04,308 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnOpenScanner/34d7236d8a6e21ffac402f86622f6a1d/.tmp/testFamily/2277260fa62c4e71bb8aa5a99ac22005 as hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnOpenScanner/34d7236d8a6e21ffac402f86622f6a1d/testFamily/2277260fa62c4e71bb8aa5a99ac22005 2024-11-27T17:39:04,308 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testNormalScanTimeoutOnOpenScannerAsync' 2024-11-27T17:39:04,317 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnOpenScanner/34d7236d8a6e21ffac402f86622f6a1d/testFamily/2277260fa62c4e71bb8aa5a99ac22005, entries=4, sequenceid=8, filesize=4.9 K 2024-11-27T17:39:04,323 DEBUG [RS_CLOSE_META-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/meta/1588230740/.tmp/ns/067626775e3140b680b6f05465e71531 is 43, key is default/ns:d/1732729078378/Put/seqid=0 2024-11-27T17:39:04,326 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 34d7236d8a6e21ffac402f86622f6a1d in 548ms, sequenceid=8, compaction requested=false 2024-11-27T17:39:04,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741857_1033 (size=5153) 2024-11-27T17:39:04,332 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnOpenScanner/34d7236d8a6e21ffac402f86622f6a1d/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-27T17:39:04,334 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732729081763.34d7236d8a6e21ffac402f86622f6a1d. 2024-11-27T17:39:04,334 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 34d7236d8a6e21ffac402f86622f6a1d: Waiting for close lock at 1732729143772Running coprocessor pre-close hooks at 1732729143773 (+1 ms)Disabling compacts and flushes for region at 1732729143773Disabling writes for close at 1732729143773Obtaining lock to block concurrent updates at 1732729143777 (+4 ms)Preparing flush snapshotting stores in 34d7236d8a6e21ffac402f86622f6a1d at 1732729143777Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScanner,,1732729081763.34d7236d8a6e21ffac402f86622f6a1d., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732729143784 (+7 ms)Flushing stores of hbase:testMetaScanTimeoutOnOpenScanner,,1732729081763.34d7236d8a6e21ffac402f86622f6a1d. at 1732729143785 (+1 ms)Flushing 34d7236d8a6e21ffac402f86622f6a1d/testFamily: creating writer at 1732729143786 (+1 ms)Flushing 34d7236d8a6e21ffac402f86622f6a1d/testFamily: appending metadata at 1732729143835 (+49 ms)Flushing 34d7236d8a6e21ffac402f86622f6a1d/testFamily: closing flushed file at 1732729143838 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@354d6022: reopening flushed file at 1732729144305 (+467 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 34d7236d8a6e21ffac402f86622f6a1d in 548ms, sequenceid=8, compaction requested=false at 1732729144326 (+21 ms)Writing region close event to WAL at 1732729144328 (+2 ms)Running coprocessor post-close hooks at 1732729144333 (+5 ms)Closed at 1732729144334 (+1 ms) 2024-11-27T17:39:04,335 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScanner,,1732729081763.34d7236d8a6e21ffac402f86622f6a1d. 2024-11-27T17:39:04,335 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 528929fce6443603ca9db935c1532830, disabling compactions & flushes 2024-11-27T17:39:04,335 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNext,,1732729113284.528929fce6443603ca9db935c1532830. 2024-11-27T17:39:04,335 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNext,,1732729113284.528929fce6443603ca9db935c1532830. 2024-11-27T17:39:04,335 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNext,,1732729113284.528929fce6443603ca9db935c1532830. after waiting 0 ms 2024-11-27T17:39:04,335 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNext,,1732729113284.528929fce6443603ca9db935c1532830. 2024-11-27T17:39:04,335 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 528929fce6443603ca9db935c1532830 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-27T17:39:04,354 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnNext/528929fce6443603ca9db935c1532830/.tmp/testFamily/84e57a1364924e589c9cb27a36c748e8 is 61, key is row-0/testFamily:testQualifier/1732729116340/Put/seqid=0 2024-11-27T17:39:04,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741858_1034 (size=5049) 2024-11-27T17:39:04,359 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnNext/528929fce6443603ca9db935c1532830/.tmp/testFamily/84e57a1364924e589c9cb27a36c748e8 2024-11-27T17:39:04,369 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnNext/528929fce6443603ca9db935c1532830/.tmp/testFamily/84e57a1364924e589c9cb27a36c748e8 as hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnNext/528929fce6443603ca9db935c1532830/testFamily/84e57a1364924e589c9cb27a36c748e8 2024-11-27T17:39:04,376 DEBUG [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1351): Waiting on 0105071c31a7ce20d864bf590e719c24, 1588230740, 3fd0994b61fc0961c034fe9da87dbbfd, 528929fce6443603ca9db935c1532830, 6de5dbf4f8dbf9b8abcc9963e93407d3, 87e3e592443f2e8ffc03e943933578f6, 8ebfd4c85458b86adf7a32ac08f189f7, a46374dfd125bdd1346773a9c685bf28, a82ab4b705fac222998b7c9c55b9d4a6, f048b20286d5f531019fd9a36448940a 2024-11-27T17:39:04,378 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnNext/528929fce6443603ca9db935c1532830/testFamily/84e57a1364924e589c9cb27a36c748e8, entries=4, sequenceid=8, filesize=4.9 K 2024-11-27T17:39:04,380 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 528929fce6443603ca9db935c1532830 in 45ms, sequenceid=8, compaction requested=false 2024-11-27T17:39:04,386 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnNext/528929fce6443603ca9db935c1532830/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-27T17:39:04,387 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNext,,1732729113284.528929fce6443603ca9db935c1532830. 2024-11-27T17:39:04,387 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 528929fce6443603ca9db935c1532830: Waiting for close lock at 1732729144335Running coprocessor pre-close hooks at 1732729144335Disabling compacts and flushes for region at 1732729144335Disabling writes for close at 1732729144335Obtaining lock to block concurrent updates at 1732729144335Preparing flush snapshotting stores in 528929fce6443603ca9db935c1532830 at 1732729144335Finished memstore snapshotting testNormalScanTimeoutOnNext,,1732729113284.528929fce6443603ca9db935c1532830., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732729144336 (+1 ms)Flushing stores of testNormalScanTimeoutOnNext,,1732729113284.528929fce6443603ca9db935c1532830. at 1732729144336Flushing 528929fce6443603ca9db935c1532830/testFamily: creating writer at 1732729144337 (+1 ms)Flushing 528929fce6443603ca9db935c1532830/testFamily: appending metadata at 1732729144353 (+16 ms)Flushing 528929fce6443603ca9db935c1532830/testFamily: closing flushed file at 1732729144354 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14fee128: reopening flushed file at 1732729144368 (+14 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 528929fce6443603ca9db935c1532830 in 45ms, sequenceid=8, compaction requested=false at 1732729144380 (+12 ms)Writing region close event to WAL at 1732729144381 (+1 ms)Running coprocessor post-close hooks at 1732729144386 (+5 ms)Closed at 1732729144386 2024-11-27T17:39:04,387 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNext,,1732729113284.528929fce6443603ca9db935c1532830. 2024-11-27T17:39:04,387 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6de5dbf4f8dbf9b8abcc9963e93407d3, disabling compactions & flushes 2024-11-27T17:39:04,387 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3. 2024-11-27T17:39:04,387 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3. 2024-11-27T17:39:04,387 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3. after waiting 0 ms 2024-11-27T17:39:04,387 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3. 2024-11-27T17:39:04,387 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 6de5dbf4f8dbf9b8abcc9963e93407d3 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-27T17:39:04,404 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnOpenScannerAsync/6de5dbf4f8dbf9b8abcc9963e93407d3/.tmp/testFamily/515689fccb454ad8bf6c2902ee5142ce is 61, key is row-0/testFamily:testQualifier/1732729140658/Put/seqid=0 2024-11-27T17:39:04,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741859_1035 (size=5049) 2024-11-27T17:39:04,411 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnOpenScannerAsync/6de5dbf4f8dbf9b8abcc9963e93407d3/.tmp/testFamily/515689fccb454ad8bf6c2902ee5142ce 2024-11-27T17:39:04,422 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnOpenScannerAsync/6de5dbf4f8dbf9b8abcc9963e93407d3/.tmp/testFamily/515689fccb454ad8bf6c2902ee5142ce as hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnOpenScannerAsync/6de5dbf4f8dbf9b8abcc9963e93407d3/testFamily/515689fccb454ad8bf6c2902ee5142ce 2024-11-27T17:39:04,432 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnOpenScannerAsync/6de5dbf4f8dbf9b8abcc9963e93407d3/testFamily/515689fccb454ad8bf6c2902ee5142ce, entries=4, sequenceid=8, filesize=4.9 K 2024-11-27T17:39:04,434 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6de5dbf4f8dbf9b8abcc9963e93407d3 in 47ms, sequenceid=8, compaction requested=false 2024-11-27T17:39:04,439 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnOpenScannerAsync/6de5dbf4f8dbf9b8abcc9963e93407d3/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-27T17:39:04,440 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3. 2024-11-27T17:39:04,441 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6de5dbf4f8dbf9b8abcc9963e93407d3: Waiting for close lock at 1732729144387Running coprocessor pre-close hooks at 1732729144387Disabling compacts and flushes for region at 1732729144387Disabling writes for close at 1732729144387Obtaining lock to block concurrent updates at 1732729144387Preparing flush snapshotting stores in 6de5dbf4f8dbf9b8abcc9963e93407d3 at 1732729144387Finished memstore snapshotting testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732729144388 (+1 ms)Flushing stores of testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3. at 1732729144389 (+1 ms)Flushing 6de5dbf4f8dbf9b8abcc9963e93407d3/testFamily: creating writer at 1732729144389Flushing 6de5dbf4f8dbf9b8abcc9963e93407d3/testFamily: appending metadata at 1732729144404 (+15 ms)Flushing 6de5dbf4f8dbf9b8abcc9963e93407d3/testFamily: closing flushed file at 1732729144404Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46f6dc64: reopening flushed file at 1732729144420 (+16 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 6de5dbf4f8dbf9b8abcc9963e93407d3 in 47ms, sequenceid=8, compaction requested=false at 1732729144434 (+14 ms)Writing region close event to WAL at 1732729144435 (+1 ms)Running coprocessor post-close hooks at 1732729144440 (+5 ms)Closed at 1732729144440 2024-11-27T17:39:04,441 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScannerAsync,,1732729137601.6de5dbf4f8dbf9b8abcc9963e93407d3. 2024-11-27T17:39:04,441 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a82ab4b705fac222998b7c9c55b9d4a6, disabling compactions & flushes 2024-11-27T17:39:04,441 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextException,,1732729097970.a82ab4b705fac222998b7c9c55b9d4a6. 2024-11-27T17:39:04,441 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextException,,1732729097970.a82ab4b705fac222998b7c9c55b9d4a6. 2024-11-27T17:39:04,441 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextException,,1732729097970.a82ab4b705fac222998b7c9c55b9d4a6. after waiting 0 ms 2024-11-27T17:39:04,441 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextException,,1732729097970.a82ab4b705fac222998b7c9c55b9d4a6. 2024-11-27T17:39:04,441 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing a82ab4b705fac222998b7c9c55b9d4a6 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-27T17:39:04,459 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testRetryOutOfOrderScannerNextException/a82ab4b705fac222998b7c9c55b9d4a6/.tmp/testFamily/e0272576d0c94e899a2148a0ed5c860d is 61, key is row-0/testFamily:testQualifier/1732729101016/Put/seqid=0 2024-11-27T17:39:04,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741860_1036 (size=5049) 2024-11-27T17:39:04,541 INFO [regionserver/8f0673442175:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-27T17:39:04,542 INFO [regionserver/8f0673442175:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-27T17:39:04,576 DEBUG [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1351): Waiting on 0105071c31a7ce20d864bf590e719c24, 1588230740, 3fd0994b61fc0961c034fe9da87dbbfd, 87e3e592443f2e8ffc03e943933578f6, 8ebfd4c85458b86adf7a32ac08f189f7, a46374dfd125bdd1346773a9c685bf28, a82ab4b705fac222998b7c9c55b9d4a6, f048b20286d5f531019fd9a36448940a 2024-11-27T17:39:04,730 INFO [RS_CLOSE_META-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/meta/1588230740/.tmp/ns/067626775e3140b680b6f05465e71531 2024-11-27T17:39:04,760 DEBUG [RS_CLOSE_META-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/meta/1588230740/.tmp/table/de40d63c348448c19f38738ff787cef8 is 80, key is testRetryOutOfOrderScannerNextExceptionAsync/table:state/1732729079856/Put/seqid=0 2024-11-27T17:39:04,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741861_1037 (size=6699) 2024-11-27T17:39:04,776 INFO [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1321): Waiting on 8 regions to close 2024-11-27T17:39:04,776 DEBUG [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1325): Online Regions={a82ab4b705fac222998b7c9c55b9d4a6=testRetryOutOfOrderScannerNextException,,1732729097970.a82ab4b705fac222998b7c9c55b9d4a6., 3fd0994b61fc0961c034fe9da87dbbfd=testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd., 87e3e592443f2e8ffc03e943933578f6=testNormalScanTimeoutOnNextAsync,,1732729107187.87e3e592443f2e8ffc03e943933578f6., 0105071c31a7ce20d864bf590e719c24=hbase:testMetaScanTimeoutOnNext,,1732729128504.0105071c31a7ce20d864bf590e719c24., 1588230740=hbase:meta,,1.1588230740, a46374dfd125bdd1346773a9c685bf28=testRetryOutOfOrderScannerNextExceptionAsync,,1732729078582.a46374dfd125bdd1346773a9c685bf28., 8ebfd4c85458b86adf7a32ac08f189f7=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732729119390.8ebfd4c85458b86adf7a32ac08f189f7., f048b20286d5f531019fd9a36448940a=hbase:testMetaScanTimeoutOnNextAsync,,1732729090880.f048b20286d5f531019fd9a36448940a.} 2024-11-27T17:39:04,777 DEBUG [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1351): Waiting on 0105071c31a7ce20d864bf590e719c24, 1588230740, 3fd0994b61fc0961c034fe9da87dbbfd, 87e3e592443f2e8ffc03e943933578f6, 8ebfd4c85458b86adf7a32ac08f189f7, a46374dfd125bdd1346773a9c685bf28, a82ab4b705fac222998b7c9c55b9d4a6, f048b20286d5f531019fd9a36448940a 2024-11-27T17:39:04,866 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testRetryOutOfOrderScannerNextException/a82ab4b705fac222998b7c9c55b9d4a6/.tmp/testFamily/e0272576d0c94e899a2148a0ed5c860d 2024-11-27T17:39:04,881 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testRetryOutOfOrderScannerNextException/a82ab4b705fac222998b7c9c55b9d4a6/.tmp/testFamily/e0272576d0c94e899a2148a0ed5c860d as hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testRetryOutOfOrderScannerNextException/a82ab4b705fac222998b7c9c55b9d4a6/testFamily/e0272576d0c94e899a2148a0ed5c860d 2024-11-27T17:39:04,891 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testRetryOutOfOrderScannerNextException/a82ab4b705fac222998b7c9c55b9d4a6/testFamily/e0272576d0c94e899a2148a0ed5c860d, entries=4, sequenceid=8, filesize=4.9 K 2024-11-27T17:39:04,892 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a82ab4b705fac222998b7c9c55b9d4a6 in 451ms, sequenceid=8, compaction requested=false 2024-11-27T17:39:04,898 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testRetryOutOfOrderScannerNextException/a82ab4b705fac222998b7c9c55b9d4a6/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-27T17:39:04,899 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextException,,1732729097970.a82ab4b705fac222998b7c9c55b9d4a6. 2024-11-27T17:39:04,899 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a82ab4b705fac222998b7c9c55b9d4a6: Waiting for close lock at 1732729144441Running coprocessor pre-close hooks at 1732729144441Disabling compacts and flushes for region at 1732729144441Disabling writes for close at 1732729144441Obtaining lock to block concurrent updates at 1732729144441Preparing flush snapshotting stores in a82ab4b705fac222998b7c9c55b9d4a6 at 1732729144441Finished memstore snapshotting testRetryOutOfOrderScannerNextException,,1732729097970.a82ab4b705fac222998b7c9c55b9d4a6., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732729144441Flushing stores of testRetryOutOfOrderScannerNextException,,1732729097970.a82ab4b705fac222998b7c9c55b9d4a6. at 1732729144442 (+1 ms)Flushing a82ab4b705fac222998b7c9c55b9d4a6/testFamily: creating writer at 1732729144442Flushing a82ab4b705fac222998b7c9c55b9d4a6/testFamily: appending metadata at 1732729144458 (+16 ms)Flushing a82ab4b705fac222998b7c9c55b9d4a6/testFamily: closing flushed file at 1732729144458Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d810d20: reopening flushed file at 1732729144880 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a82ab4b705fac222998b7c9c55b9d4a6 in 451ms, sequenceid=8, compaction requested=false at 1732729144892 (+12 ms)Writing region close event to WAL at 1732729144893 (+1 ms)Running coprocessor post-close hooks at 1732729144899 (+6 ms)Closed at 1732729144899 2024-11-27T17:39:04,900 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextException,,1732729097970.a82ab4b705fac222998b7c9c55b9d4a6. 2024-11-27T17:39:04,900 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3fd0994b61fc0961c034fe9da87dbbfd, disabling compactions & flushes 2024-11-27T17:39:04,900 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd. 2024-11-27T17:39:04,900 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd. 2024-11-27T17:39:04,900 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd. after waiting 0 ms 2024-11-27T17:39:04,900 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd. 2024-11-27T17:39:04,900 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 3fd0994b61fc0961c034fe9da87dbbfd 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-27T17:39:04,920 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnOpenScanner/3fd0994b61fc0961c034fe9da87dbbfd/.tmp/testFamily/f8d71dca37774629a3b335ddd60d539e is 61, key is row-0/testFamily:testQualifier/1732729104138/Put/seqid=0 2024-11-27T17:39:04,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741862_1038 (size=5049) 2024-11-27T17:39:04,977 DEBUG [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1351): Waiting on 0105071c31a7ce20d864bf590e719c24, 1588230740, 3fd0994b61fc0961c034fe9da87dbbfd, 87e3e592443f2e8ffc03e943933578f6, 8ebfd4c85458b86adf7a32ac08f189f7, a46374dfd125bdd1346773a9c685bf28, f048b20286d5f531019fd9a36448940a 2024-11-27T17:39:05,168 INFO [RS_CLOSE_META-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.33 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/meta/1588230740/.tmp/table/de40d63c348448c19f38738ff787cef8 2024-11-27T17:39:05,177 DEBUG [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1351): Waiting on 0105071c31a7ce20d864bf590e719c24, 1588230740, 3fd0994b61fc0961c034fe9da87dbbfd, 87e3e592443f2e8ffc03e943933578f6, 8ebfd4c85458b86adf7a32ac08f189f7, a46374dfd125bdd1346773a9c685bf28, f048b20286d5f531019fd9a36448940a 2024-11-27T17:39:05,186 DEBUG [RS_CLOSE_META-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/meta/1588230740/.tmp/info/67022caf71aa4ad3bed7ed326a6a5767 as hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/meta/1588230740/info/67022caf71aa4ad3bed7ed326a6a5767 2024-11-27T17:39:05,195 INFO [RS_CLOSE_META-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/meta/1588230740/info/67022caf71aa4ad3bed7ed326a6a5767, entries=100, sequenceid=56, filesize=20.1 K 2024-11-27T17:39:05,197 DEBUG [RS_CLOSE_META-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/meta/1588230740/.tmp/ns/067626775e3140b680b6f05465e71531 as hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/meta/1588230740/ns/067626775e3140b680b6f05465e71531 2024-11-27T17:39:05,206 INFO [RS_CLOSE_META-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/meta/1588230740/ns/067626775e3140b680b6f05465e71531, entries=2, sequenceid=56, filesize=5.0 K 2024-11-27T17:39:05,208 DEBUG [RS_CLOSE_META-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/meta/1588230740/.tmp/table/de40d63c348448c19f38738ff787cef8 as hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/meta/1588230740/table/de40d63c348448c19f38738ff787cef8 2024-11-27T17:39:05,216 INFO [RS_CLOSE_META-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/meta/1588230740/table/de40d63c348448c19f38738ff787cef8, entries=20, sequenceid=56, filesize=6.5 K 2024-11-27T17:39:05,217 INFO [RS_CLOSE_META-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1443ms, sequenceid=56, compaction requested=false 2024-11-27T17:39:05,223 DEBUG [RS_CLOSE_META-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/meta/1588230740/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=1 2024-11-27T17:39:05,224 DEBUG [RS_CLOSE_META-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T17:39:05,224 INFO [RS_CLOSE_META-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T17:39:05,224 DEBUG [RS_CLOSE_META-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732729143774Running coprocessor pre-close hooks at 1732729143774Disabling compacts and flushes for region at 1732729143774Disabling writes for close at 1732729143774Obtaining lock to block concurrent updates at 1732729143777 (+3 ms)Preparing flush snapshotting stores in 1588230740 at 1732729143777Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=16053, getHeapSize=26232, getOffHeapSize=0, getCellsCount=122 at 1732729143784 (+7 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732729143785 (+1 ms)Flushing 1588230740/info: creating writer at 1732729143786 (+1 ms)Flushing 1588230740/info: appending metadata at 1732729143841 (+55 ms)Flushing 1588230740/info: closing flushed file at 1732729143841Flushing 1588230740/ns: creating writer at 1732729144305 (+464 ms)Flushing 1588230740/ns: appending metadata at 1732729144322 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1732729144322Flushing 1588230740/table: creating writer at 1732729144744 (+422 ms)Flushing 1588230740/table: appending metadata at 1732729144759 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732729144759Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f3399d6: reopening flushed file at 1732729145184 (+425 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b1fb6a6: reopening flushed file at 1732729145195 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25419132: reopening flushed file at 1732729145207 (+12 ms)Finished flush of dataSize ~15.68 KB/16053, heapSize ~25.38 KB/25992, currentSize=0 B/0 for 1588230740 in 1443ms, sequenceid=56, compaction requested=false at 1732729145217 (+10 ms)Writing region close event to WAL at 1732729145219 (+2 ms)Running coprocessor post-close hooks at 1732729145224 (+5 ms)Closed at 1732729145224 2024-11-27T17:39:05,224 DEBUG [RS_CLOSE_META-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-27T17:39:05,326 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnOpenScanner/3fd0994b61fc0961c034fe9da87dbbfd/.tmp/testFamily/f8d71dca37774629a3b335ddd60d539e 2024-11-27T17:39:05,338 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnOpenScanner/3fd0994b61fc0961c034fe9da87dbbfd/.tmp/testFamily/f8d71dca37774629a3b335ddd60d539e as hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnOpenScanner/3fd0994b61fc0961c034fe9da87dbbfd/testFamily/f8d71dca37774629a3b335ddd60d539e 2024-11-27T17:39:05,349 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnOpenScanner/3fd0994b61fc0961c034fe9da87dbbfd/testFamily/f8d71dca37774629a3b335ddd60d539e, entries=4, sequenceid=8, filesize=4.9 K 2024-11-27T17:39:05,350 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3fd0994b61fc0961c034fe9da87dbbfd in 450ms, sequenceid=8, compaction requested=false 2024-11-27T17:39:05,355 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnOpenScanner/3fd0994b61fc0961c034fe9da87dbbfd/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-27T17:39:05,356 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd. 2024-11-27T17:39:05,356 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3fd0994b61fc0961c034fe9da87dbbfd: Waiting for close lock at 1732729144900Running coprocessor pre-close hooks at 1732729144900Disabling compacts and flushes for region at 1732729144900Disabling writes for close at 1732729144900Obtaining lock to block concurrent updates at 1732729144900Preparing flush snapshotting stores in 3fd0994b61fc0961c034fe9da87dbbfd at 1732729144900Finished memstore snapshotting testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732729144901 (+1 ms)Flushing stores of testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd. at 1732729144901Flushing 3fd0994b61fc0961c034fe9da87dbbfd/testFamily: creating writer at 1732729144901Flushing 3fd0994b61fc0961c034fe9da87dbbfd/testFamily: appending metadata at 1732729144920 (+19 ms)Flushing 3fd0994b61fc0961c034fe9da87dbbfd/testFamily: closing flushed file at 1732729144920Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a2cf9c5: reopening flushed file at 1732729145337 (+417 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 3fd0994b61fc0961c034fe9da87dbbfd in 450ms, sequenceid=8, compaction requested=false at 1732729145350 (+13 ms)Writing region close event to WAL at 1732729145351 (+1 ms)Running coprocessor post-close hooks at 1732729145356 (+5 ms)Closed at 1732729145356 2024-11-27T17:39:05,356 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnOpenScanner,,1732729101070.3fd0994b61fc0961c034fe9da87dbbfd. 2024-11-27T17:39:05,356 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 87e3e592443f2e8ffc03e943933578f6, disabling compactions & flushes 2024-11-27T17:39:05,357 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testNormalScanTimeoutOnNextAsync,,1732729107187.87e3e592443f2e8ffc03e943933578f6. 2024-11-27T17:39:05,357 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testNormalScanTimeoutOnNextAsync,,1732729107187.87e3e592443f2e8ffc03e943933578f6. 2024-11-27T17:39:05,357 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testNormalScanTimeoutOnNextAsync,,1732729107187.87e3e592443f2e8ffc03e943933578f6. after waiting 0 ms 2024-11-27T17:39:05,357 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testNormalScanTimeoutOnNextAsync,,1732729107187.87e3e592443f2e8ffc03e943933578f6. 2024-11-27T17:39:05,357 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 87e3e592443f2e8ffc03e943933578f6 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-27T17:39:05,373 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnNextAsync/87e3e592443f2e8ffc03e943933578f6/.tmp/testFamily/203ac8eea4824bc192a7631ff7c7616c is 61, key is row-0/testFamily:testQualifier/1732729110244/Put/seqid=0 2024-11-27T17:39:05,377 DEBUG [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1351): Waiting on 0105071c31a7ce20d864bf590e719c24, 87e3e592443f2e8ffc03e943933578f6, 8ebfd4c85458b86adf7a32ac08f189f7, a46374dfd125bdd1346773a9c685bf28, f048b20286d5f531019fd9a36448940a 2024-11-27T17:39:05,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741863_1039 (size=5049) 2024-11-27T17:39:05,578 DEBUG [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1351): Waiting on 0105071c31a7ce20d864bf590e719c24, 87e3e592443f2e8ffc03e943933578f6, 8ebfd4c85458b86adf7a32ac08f189f7, a46374dfd125bdd1346773a9c685bf28, f048b20286d5f531019fd9a36448940a 2024-11-27T17:39:05,778 INFO [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1321): Waiting on 5 regions to close 2024-11-27T17:39:05,778 DEBUG [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1325): Online Regions={87e3e592443f2e8ffc03e943933578f6=testNormalScanTimeoutOnNextAsync,,1732729107187.87e3e592443f2e8ffc03e943933578f6., 0105071c31a7ce20d864bf590e719c24=hbase:testMetaScanTimeoutOnNext,,1732729128504.0105071c31a7ce20d864bf590e719c24., a46374dfd125bdd1346773a9c685bf28=testRetryOutOfOrderScannerNextExceptionAsync,,1732729078582.a46374dfd125bdd1346773a9c685bf28., 8ebfd4c85458b86adf7a32ac08f189f7=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732729119390.8ebfd4c85458b86adf7a32ac08f189f7., f048b20286d5f531019fd9a36448940a=hbase:testMetaScanTimeoutOnNextAsync,,1732729090880.f048b20286d5f531019fd9a36448940a.} 2024-11-27T17:39:05,778 DEBUG [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1351): Waiting on 0105071c31a7ce20d864bf590e719c24, 87e3e592443f2e8ffc03e943933578f6, 8ebfd4c85458b86adf7a32ac08f189f7, a46374dfd125bdd1346773a9c685bf28, f048b20286d5f531019fd9a36448940a 2024-11-27T17:39:05,779 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnNextAsync/87e3e592443f2e8ffc03e943933578f6/.tmp/testFamily/203ac8eea4824bc192a7631ff7c7616c 2024-11-27T17:39:05,839 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnNextAsync/87e3e592443f2e8ffc03e943933578f6/.tmp/testFamily/203ac8eea4824bc192a7631ff7c7616c as hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnNextAsync/87e3e592443f2e8ffc03e943933578f6/testFamily/203ac8eea4824bc192a7631ff7c7616c 2024-11-27T17:39:05,848 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnNextAsync/87e3e592443f2e8ffc03e943933578f6/testFamily/203ac8eea4824bc192a7631ff7c7616c, entries=4, sequenceid=8, filesize=4.9 K 2024-11-27T17:39:05,849 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 87e3e592443f2e8ffc03e943933578f6 in 492ms, sequenceid=8, compaction requested=false 2024-11-27T17:39:05,854 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testNormalScanTimeoutOnNextAsync/87e3e592443f2e8ffc03e943933578f6/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-27T17:39:05,855 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testNormalScanTimeoutOnNextAsync,,1732729107187.87e3e592443f2e8ffc03e943933578f6. 2024-11-27T17:39:05,855 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 87e3e592443f2e8ffc03e943933578f6: Waiting for close lock at 1732729145356Running coprocessor pre-close hooks at 1732729145356Disabling compacts and flushes for region at 1732729145356Disabling writes for close at 1732729145357 (+1 ms)Obtaining lock to block concurrent updates at 1732729145357Preparing flush snapshotting stores in 87e3e592443f2e8ffc03e943933578f6 at 1732729145357Finished memstore snapshotting testNormalScanTimeoutOnNextAsync,,1732729107187.87e3e592443f2e8ffc03e943933578f6., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732729145357Flushing stores of testNormalScanTimeoutOnNextAsync,,1732729107187.87e3e592443f2e8ffc03e943933578f6. at 1732729145358 (+1 ms)Flushing 87e3e592443f2e8ffc03e943933578f6/testFamily: creating writer at 1732729145358Flushing 87e3e592443f2e8ffc03e943933578f6/testFamily: appending metadata at 1732729145372 (+14 ms)Flushing 87e3e592443f2e8ffc03e943933578f6/testFamily: closing flushed file at 1732729145372Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33b5f91c: reopening flushed file at 1732729145836 (+464 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 87e3e592443f2e8ffc03e943933578f6 in 492ms, sequenceid=8, compaction requested=false at 1732729145849 (+13 ms)Writing region close event to WAL at 1732729145850 (+1 ms)Running coprocessor post-close hooks at 1732729145855 (+5 ms)Closed at 1732729145855 2024-11-27T17:39:05,855 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testNormalScanTimeoutOnNextAsync,,1732729107187.87e3e592443f2e8ffc03e943933578f6. 2024-11-27T17:39:05,855 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0105071c31a7ce20d864bf590e719c24, disabling compactions & flushes 2024-11-27T17:39:05,855 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNext,,1732729128504.0105071c31a7ce20d864bf590e719c24. 2024-11-27T17:39:05,855 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNext,,1732729128504.0105071c31a7ce20d864bf590e719c24. 2024-11-27T17:39:05,855 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNext,,1732729128504.0105071c31a7ce20d864bf590e719c24. after waiting 0 ms 2024-11-27T17:39:05,855 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNext,,1732729128504.0105071c31a7ce20d864bf590e719c24. 2024-11-27T17:39:05,856 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 0105071c31a7ce20d864bf590e719c24 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-27T17:39:05,874 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnNext/0105071c31a7ce20d864bf590e719c24/.tmp/testFamily/e54eb06de7884c149bcfa436ebff93f6 is 61, key is row-0/testFamily:testQualifier/1732729131552/Put/seqid=0 2024-11-27T17:39:05,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741864_1040 (size=5049) 2024-11-27T17:39:05,979 DEBUG [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1351): Waiting on 0105071c31a7ce20d864bf590e719c24, 8ebfd4c85458b86adf7a32ac08f189f7, a46374dfd125bdd1346773a9c685bf28, f048b20286d5f531019fd9a36448940a 2024-11-27T17:39:06,179 DEBUG [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1351): Waiting on 0105071c31a7ce20d864bf590e719c24, 8ebfd4c85458b86adf7a32ac08f189f7, a46374dfd125bdd1346773a9c685bf28, f048b20286d5f531019fd9a36448940a 2024-11-27T17:39:06,281 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnNext/0105071c31a7ce20d864bf590e719c24/.tmp/testFamily/e54eb06de7884c149bcfa436ebff93f6 2024-11-27T17:39:06,297 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnNext/0105071c31a7ce20d864bf590e719c24/.tmp/testFamily/e54eb06de7884c149bcfa436ebff93f6 as hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnNext/0105071c31a7ce20d864bf590e719c24/testFamily/e54eb06de7884c149bcfa436ebff93f6 2024-11-27T17:39:06,306 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnNext/0105071c31a7ce20d864bf590e719c24/testFamily/e54eb06de7884c149bcfa436ebff93f6, entries=4, sequenceid=8, filesize=4.9 K 2024-11-27T17:39:06,308 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0105071c31a7ce20d864bf590e719c24 in 452ms, sequenceid=8, compaction requested=false 2024-11-27T17:39:06,313 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnNext/0105071c31a7ce20d864bf590e719c24/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-27T17:39:06,314 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNext,,1732729128504.0105071c31a7ce20d864bf590e719c24. 2024-11-27T17:39:06,314 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0105071c31a7ce20d864bf590e719c24: Waiting for close lock at 1732729145855Running coprocessor pre-close hooks at 1732729145855Disabling compacts and flushes for region at 1732729145855Disabling writes for close at 1732729145855Obtaining lock to block concurrent updates at 1732729145856 (+1 ms)Preparing flush snapshotting stores in 0105071c31a7ce20d864bf590e719c24 at 1732729145856Finished memstore snapshotting hbase:testMetaScanTimeoutOnNext,,1732729128504.0105071c31a7ce20d864bf590e719c24., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732729145856Flushing stores of hbase:testMetaScanTimeoutOnNext,,1732729128504.0105071c31a7ce20d864bf590e719c24. at 1732729145857 (+1 ms)Flushing 0105071c31a7ce20d864bf590e719c24/testFamily: creating writer at 1732729145857Flushing 0105071c31a7ce20d864bf590e719c24/testFamily: appending metadata at 1732729145874 (+17 ms)Flushing 0105071c31a7ce20d864bf590e719c24/testFamily: closing flushed file at 1732729145874Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47729edd: reopening flushed file at 1732729146295 (+421 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 0105071c31a7ce20d864bf590e719c24 in 452ms, sequenceid=8, compaction requested=false at 1732729146308 (+13 ms)Writing region close event to WAL at 1732729146308Running coprocessor post-close hooks at 1732729146314 (+6 ms)Closed at 1732729146314 2024-11-27T17:39:06,314 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNext,,1732729128504.0105071c31a7ce20d864bf590e719c24. 2024-11-27T17:39:06,314 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a46374dfd125bdd1346773a9c685bf28, disabling compactions & flushes 2024-11-27T17:39:06,314 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testRetryOutOfOrderScannerNextExceptionAsync,,1732729078582.a46374dfd125bdd1346773a9c685bf28. 2024-11-27T17:39:06,314 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732729078582.a46374dfd125bdd1346773a9c685bf28. 2024-11-27T17:39:06,314 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testRetryOutOfOrderScannerNextExceptionAsync,,1732729078582.a46374dfd125bdd1346773a9c685bf28. after waiting 0 ms 2024-11-27T17:39:06,314 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testRetryOutOfOrderScannerNextExceptionAsync,,1732729078582.a46374dfd125bdd1346773a9c685bf28. 2024-11-27T17:39:06,315 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing a46374dfd125bdd1346773a9c685bf28 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-27T17:39:06,331 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testRetryOutOfOrderScannerNextExceptionAsync/a46374dfd125bdd1346773a9c685bf28/.tmp/testFamily/aef562ced3ab4dcc84e703693c5147df is 61, key is row-0/testFamily:testQualifier/1732729081688/Put/seqid=0 2024-11-27T17:39:06,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741865_1041 (size=5049) 2024-11-27T17:39:06,380 DEBUG [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1351): Waiting on 8ebfd4c85458b86adf7a32ac08f189f7, a46374dfd125bdd1346773a9c685bf28, f048b20286d5f531019fd9a36448940a 2024-11-27T17:39:06,580 DEBUG [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1351): Waiting on 8ebfd4c85458b86adf7a32ac08f189f7, a46374dfd125bdd1346773a9c685bf28, f048b20286d5f531019fd9a36448940a 2024-11-27T17:39:06,738 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testRetryOutOfOrderScannerNextExceptionAsync/a46374dfd125bdd1346773a9c685bf28/.tmp/testFamily/aef562ced3ab4dcc84e703693c5147df 2024-11-27T17:39:06,750 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testRetryOutOfOrderScannerNextExceptionAsync/a46374dfd125bdd1346773a9c685bf28/.tmp/testFamily/aef562ced3ab4dcc84e703693c5147df as hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testRetryOutOfOrderScannerNextExceptionAsync/a46374dfd125bdd1346773a9c685bf28/testFamily/aef562ced3ab4dcc84e703693c5147df 2024-11-27T17:39:06,757 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testRetryOutOfOrderScannerNextExceptionAsync/a46374dfd125bdd1346773a9c685bf28/testFamily/aef562ced3ab4dcc84e703693c5147df, entries=4, sequenceid=8, filesize=4.9 K 2024-11-27T17:39:06,758 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a46374dfd125bdd1346773a9c685bf28 in 444ms, sequenceid=8, compaction requested=false 2024-11-27T17:39:06,763 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/default/testRetryOutOfOrderScannerNextExceptionAsync/a46374dfd125bdd1346773a9c685bf28/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-27T17:39:06,763 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732729078582.a46374dfd125bdd1346773a9c685bf28. 2024-11-27T17:39:06,763 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a46374dfd125bdd1346773a9c685bf28: Waiting for close lock at 1732729146314Running coprocessor pre-close hooks at 1732729146314Disabling compacts and flushes for region at 1732729146314Disabling writes for close at 1732729146314Obtaining lock to block concurrent updates at 1732729146315 (+1 ms)Preparing flush snapshotting stores in a46374dfd125bdd1346773a9c685bf28 at 1732729146315Finished memstore snapshotting testRetryOutOfOrderScannerNextExceptionAsync,,1732729078582.a46374dfd125bdd1346773a9c685bf28., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732729146315Flushing stores of testRetryOutOfOrderScannerNextExceptionAsync,,1732729078582.a46374dfd125bdd1346773a9c685bf28. at 1732729146315Flushing a46374dfd125bdd1346773a9c685bf28/testFamily: creating writer at 1732729146316 (+1 ms)Flushing a46374dfd125bdd1346773a9c685bf28/testFamily: appending metadata at 1732729146331 (+15 ms)Flushing a46374dfd125bdd1346773a9c685bf28/testFamily: closing flushed file at 1732729146331Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@654e23b5: reopening flushed file at 1732729146749 (+418 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for a46374dfd125bdd1346773a9c685bf28 in 444ms, sequenceid=8, compaction requested=false at 1732729146758 (+9 ms)Writing region close event to WAL at 1732729146759 (+1 ms)Running coprocessor post-close hooks at 1732729146763 (+4 ms)Closed at 1732729146763 2024-11-27T17:39:06,764 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testRetryOutOfOrderScannerNextExceptionAsync,,1732729078582.a46374dfd125bdd1346773a9c685bf28. 2024-11-27T17:39:06,764 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8ebfd4c85458b86adf7a32ac08f189f7, disabling compactions & flushes 2024-11-27T17:39:06,764 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732729119390.8ebfd4c85458b86adf7a32ac08f189f7. 2024-11-27T17:39:06,764 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732729119390.8ebfd4c85458b86adf7a32ac08f189f7. 2024-11-27T17:39:06,764 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732729119390.8ebfd4c85458b86adf7a32ac08f189f7. after waiting 0 ms 2024-11-27T17:39:06,764 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732729119390.8ebfd4c85458b86adf7a32ac08f189f7. 2024-11-27T17:39:06,764 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 8ebfd4c85458b86adf7a32ac08f189f7 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-27T17:39:06,780 INFO [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-27T17:39:06,780 DEBUG [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1325): Online Regions={8ebfd4c85458b86adf7a32ac08f189f7=hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732729119390.8ebfd4c85458b86adf7a32ac08f189f7., f048b20286d5f531019fd9a36448940a=hbase:testMetaScanTimeoutOnNextAsync,,1732729090880.f048b20286d5f531019fd9a36448940a.} 2024-11-27T17:39:06,781 DEBUG [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1351): Waiting on 8ebfd4c85458b86adf7a32ac08f189f7, f048b20286d5f531019fd9a36448940a 2024-11-27T17:39:06,782 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/8ebfd4c85458b86adf7a32ac08f189f7/.tmp/testFamily/eda4b12e8e9549b6b51fe474a753701d is 61, key is row-0/testFamily:testQualifier/1732729122452/Put/seqid=0 2024-11-27T17:39:06,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741866_1042 (size=5049) 2024-11-27T17:39:06,981 DEBUG [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1351): Waiting on 8ebfd4c85458b86adf7a32ac08f189f7, f048b20286d5f531019fd9a36448940a 2024-11-27T17:39:07,181 DEBUG [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1351): Waiting on 8ebfd4c85458b86adf7a32ac08f189f7, f048b20286d5f531019fd9a36448940a 2024-11-27T17:39:07,190 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/8ebfd4c85458b86adf7a32ac08f189f7/.tmp/testFamily/eda4b12e8e9549b6b51fe474a753701d 2024-11-27T17:39:07,206 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/8ebfd4c85458b86adf7a32ac08f189f7/.tmp/testFamily/eda4b12e8e9549b6b51fe474a753701d as hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/8ebfd4c85458b86adf7a32ac08f189f7/testFamily/eda4b12e8e9549b6b51fe474a753701d 2024-11-27T17:39:07,215 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/8ebfd4c85458b86adf7a32ac08f189f7/testFamily/eda4b12e8e9549b6b51fe474a753701d, entries=4, sequenceid=8, filesize=4.9 K 2024-11-27T17:39:07,216 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8ebfd4c85458b86adf7a32ac08f189f7 in 452ms, sequenceid=8, compaction requested=false 2024-11-27T17:39:07,221 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnOpenScannerAsync/8ebfd4c85458b86adf7a32ac08f189f7/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-27T17:39:07,222 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732729119390.8ebfd4c85458b86adf7a32ac08f189f7. 2024-11-27T17:39:07,222 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8ebfd4c85458b86adf7a32ac08f189f7: Waiting for close lock at 1732729146764Running coprocessor pre-close hooks at 1732729146764Disabling compacts and flushes for region at 1732729146764Disabling writes for close at 1732729146764Obtaining lock to block concurrent updates at 1732729146764Preparing flush snapshotting stores in 8ebfd4c85458b86adf7a32ac08f189f7 at 1732729146764Finished memstore snapshotting hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732729119390.8ebfd4c85458b86adf7a32ac08f189f7., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732729146764Flushing stores of hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732729119390.8ebfd4c85458b86adf7a32ac08f189f7. at 1732729146765 (+1 ms)Flushing 8ebfd4c85458b86adf7a32ac08f189f7/testFamily: creating writer at 1732729146765Flushing 8ebfd4c85458b86adf7a32ac08f189f7/testFamily: appending metadata at 1732729146781 (+16 ms)Flushing 8ebfd4c85458b86adf7a32ac08f189f7/testFamily: closing flushed file at 1732729146781Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5aa9304a: reopening flushed file at 1732729147204 (+423 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for 8ebfd4c85458b86adf7a32ac08f189f7 in 452ms, sequenceid=8, compaction requested=false at 1732729147216 (+12 ms)Writing region close event to WAL at 1732729147217 (+1 ms)Running coprocessor post-close hooks at 1732729147222 (+5 ms)Closed at 1732729147222 2024-11-27T17:39:07,222 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnOpenScannerAsync,,1732729119390.8ebfd4c85458b86adf7a32ac08f189f7. 2024-11-27T17:39:07,223 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f048b20286d5f531019fd9a36448940a, disabling compactions & flushes 2024-11-27T17:39:07,223 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:testMetaScanTimeoutOnNextAsync,,1732729090880.f048b20286d5f531019fd9a36448940a. 2024-11-27T17:39:07,223 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732729090880.f048b20286d5f531019fd9a36448940a. 2024-11-27T17:39:07,223 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:testMetaScanTimeoutOnNextAsync,,1732729090880.f048b20286d5f531019fd9a36448940a. after waiting 0 ms 2024-11-27T17:39:07,223 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:testMetaScanTimeoutOnNextAsync,,1732729090880.f048b20286d5f531019fd9a36448940a. 2024-11-27T17:39:07,223 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing f048b20286d5f531019fd9a36448940a 1/1 column families, dataSize=228 B heapSize=800 B 2024-11-27T17:39:07,240 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnNextAsync/f048b20286d5f531019fd9a36448940a/.tmp/testFamily/e1d672791211471389adcfc626ff0c77 is 61, key is row-0/testFamily:testQualifier/1732729091918/Put/seqid=0 2024-11-27T17:39:07,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741867_1043 (size=5049) 2024-11-27T17:39:07,382 DEBUG [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1351): Waiting on f048b20286d5f531019fd9a36448940a 2024-11-27T17:39:07,582 DEBUG [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1351): Waiting on f048b20286d5f531019fd9a36448940a 2024-11-27T17:39:07,647 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=228 B at sequenceid=8 (bloomFilter=false), to=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnNextAsync/f048b20286d5f531019fd9a36448940a/.tmp/testFamily/e1d672791211471389adcfc626ff0c77 2024-11-27T17:39:07,662 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnNextAsync/f048b20286d5f531019fd9a36448940a/.tmp/testFamily/e1d672791211471389adcfc626ff0c77 as hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnNextAsync/f048b20286d5f531019fd9a36448940a/testFamily/e1d672791211471389adcfc626ff0c77 2024-11-27T17:39:07,669 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnNextAsync/f048b20286d5f531019fd9a36448940a/testFamily/e1d672791211471389adcfc626ff0c77, entries=4, sequenceid=8, filesize=4.9 K 2024-11-27T17:39:07,671 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f048b20286d5f531019fd9a36448940a in 448ms, sequenceid=8, compaction requested=false 2024-11-27T17:39:07,677 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/data/hbase/testMetaScanTimeoutOnNextAsync/f048b20286d5f531019fd9a36448940a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-27T17:39:07,678 INFO [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732729090880.f048b20286d5f531019fd9a36448940a. 2024-11-27T17:39:07,678 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f048b20286d5f531019fd9a36448940a: Waiting for close lock at 1732729147223Running coprocessor pre-close hooks at 1732729147223Disabling compacts and flushes for region at 1732729147223Disabling writes for close at 1732729147223Obtaining lock to block concurrent updates at 1732729147223Preparing flush snapshotting stores in f048b20286d5f531019fd9a36448940a at 1732729147223Finished memstore snapshotting hbase:testMetaScanTimeoutOnNextAsync,,1732729090880.f048b20286d5f531019fd9a36448940a., syncing WAL and waiting on mvcc, flushsize=dataSize=228, getHeapSize=784, getOffHeapSize=0, getCellsCount=4 at 1732729147223Flushing stores of hbase:testMetaScanTimeoutOnNextAsync,,1732729090880.f048b20286d5f531019fd9a36448940a. at 1732729147224 (+1 ms)Flushing f048b20286d5f531019fd9a36448940a/testFamily: creating writer at 1732729147224Flushing f048b20286d5f531019fd9a36448940a/testFamily: appending metadata at 1732729147239 (+15 ms)Flushing f048b20286d5f531019fd9a36448940a/testFamily: closing flushed file at 1732729147239Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23adfe7b: reopening flushed file at 1732729147661 (+422 ms)Finished flush of dataSize ~228 B/228, heapSize ~784 B/784, currentSize=0 B/0 for f048b20286d5f531019fd9a36448940a in 448ms, sequenceid=8, compaction requested=false at 1732729147671 (+10 ms)Writing region close event to WAL at 1732729147671Running coprocessor post-close hooks at 1732729147678 (+7 ms)Closed at 1732729147678 2024-11-27T17:39:07,678 DEBUG [RS_CLOSE_REGION-regionserver/8f0673442175:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:testMetaScanTimeoutOnNextAsync,,1732729090880.f048b20286d5f531019fd9a36448940a. 2024-11-27T17:39:07,783 INFO [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(976): stopping server 8f0673442175,43899,1732729014292; all regions closed. 2024-11-27T17:39:07,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741834_1010 (size=19479) 2024-11-27T17:39:07,793 DEBUG [RS:0;8f0673442175:43899 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/oldWALs 2024-11-27T17:39:07,793 INFO [RS:0;8f0673442175:43899 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8f0673442175%2C43899%2C1732729014292.meta:.meta(num 1732729078148) 2024-11-27T17:39:07,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741833_1009 (size=20031) 2024-11-27T17:39:07,796 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/WALs/8f0673442175,43899,1732729014292/8f0673442175%2C43899%2C1732729014292.1732729077602 not finished, retry = 0 2024-11-27T17:39:07,902 DEBUG [RS:0;8f0673442175:43899 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/oldWALs 2024-11-27T17:39:07,902 INFO [RS:0;8f0673442175:43899 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8f0673442175%2C43899%2C1732729014292:(num 1732729077602) 2024-11-27T17:39:07,902 DEBUG [RS:0;8f0673442175:43899 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T17:39:07,902 INFO [RS:0;8f0673442175:43899 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T17:39:07,903 INFO [RS:0;8f0673442175:43899 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T17:39:07,903 INFO [RS:0;8f0673442175:43899 {}] hbase.ChoreService(370): Chore service for: regionserver/8f0673442175:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T17:39:07,904 INFO [RS:0;8f0673442175:43899 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T17:39:07,904 INFO [regionserver/8f0673442175:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T17:39:07,907 INFO [RS:0;8f0673442175:43899 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43899 2024-11-27T17:39:07,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46243-0x1017d5e7cd30000, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T17:39:07,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43899-0x1017d5e7cd30001, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/8f0673442175,43899,1732729014292 2024-11-27T17:39:07,965 INFO [RS:0;8f0673442175:43899 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T17:39:07,968 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [8f0673442175,43899,1732729014292] 2024-11-27T17:39:07,973 INFO [regionserver/8f0673442175:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T17:39:07,986 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/8f0673442175,43899,1732729014292 already deleted, retry=false 2024-11-27T17:39:07,986 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 8f0673442175,43899,1732729014292 expired; onlineServers=0 2024-11-27T17:39:07,987 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '8f0673442175,46243,1732729013300' ***** 2024-11-27T17:39:07,987 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-27T17:39:07,987 INFO [M:0;8f0673442175:46243 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T17:39:07,988 INFO [M:0;8f0673442175:46243 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T17:39:07,988 DEBUG [M:0;8f0673442175:46243 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-27T17:39:07,988 DEBUG [M:0;8f0673442175:46243 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-27T17:39:07,988 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-27T17:39:07,989 DEBUG [master/8f0673442175:0:becomeActiveMaster-HFileCleaner.large.0-1732729017311 {}] cleaner.HFileCleaner(306): Exit Thread[master/8f0673442175:0:becomeActiveMaster-HFileCleaner.large.0-1732729017311,5,FailOnTimeoutGroup] 2024-11-27T17:39:07,988 DEBUG [master/8f0673442175:0:becomeActiveMaster-HFileCleaner.small.0-1732729017320 {}] cleaner.HFileCleaner(306): Exit Thread[master/8f0673442175:0:becomeActiveMaster-HFileCleaner.small.0-1732729017320,5,FailOnTimeoutGroup] 2024-11-27T17:39:07,989 INFO [M:0;8f0673442175:46243 {}] hbase.ChoreService(370): Chore service for: master/8f0673442175:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-27T17:39:07,990 INFO [M:0;8f0673442175:46243 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T17:39:07,990 DEBUG [M:0;8f0673442175:46243 {}] master.HMaster(1795): Stopping service threads 2024-11-27T17:39:07,990 INFO [M:0;8f0673442175:46243 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-27T17:39:07,991 INFO [M:0;8f0673442175:46243 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T17:39:07,992 INFO [M:0;8f0673442175:46243 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T17:39:07,992 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-27T17:39:07,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46243-0x1017d5e7cd30000, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-27T17:39:07,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46243-0x1017d5e7cd30000, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T17:39:07,996 DEBUG [M:0;8f0673442175:46243 {}] zookeeper.ZKUtil(347): master:46243-0x1017d5e7cd30000, quorum=127.0.0.1:64480, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-27T17:39:07,997 WARN [M:0;8f0673442175:46243 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-27T17:39:07,998 INFO [M:0;8f0673442175:46243 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/.lastflushedseqids 2024-11-27T17:39:08,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741868_1044 (size=649) 2024-11-27T17:39:08,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43899-0x1017d5e7cd30001, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T17:39:08,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43899-0x1017d5e7cd30001, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T17:39:08,076 INFO [RS:0;8f0673442175:43899 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T17:39:08,077 INFO [RS:0;8f0673442175:43899 {}] regionserver.HRegionServer(1031): Exiting; stopping=8f0673442175,43899,1732729014292; zookeeper connection closed. 2024-11-27T17:39:08,077 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@70642c65 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@70642c65 2024-11-27T17:39:08,078 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-27T17:39:08,411 INFO [M:0;8f0673442175:46243 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T17:39:08,412 INFO [M:0;8f0673442175:46243 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-27T17:39:08,413 DEBUG [M:0;8f0673442175:46243 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T17:39:08,413 INFO [M:0;8f0673442175:46243 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T17:39:08,413 DEBUG [M:0;8f0673442175:46243 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T17:39:08,413 DEBUG [M:0;8f0673442175:46243 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T17:39:08,414 DEBUG [M:0;8f0673442175:46243 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T17:39:08,414 INFO [M:0;8f0673442175:46243 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=162.99 KB heapSize=190.92 KB 2024-11-27T17:39:08,436 DEBUG [M:0;8f0673442175:46243 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dca2e0512a3240148025de7094376231 is 82, key is hbase:meta,,1/info:regioninfo/1732729078230/Put/seqid=0 2024-11-27T17:39:08,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741869_1045 (size=5672) 2024-11-27T17:39:08,843 INFO [M:0;8f0673442175:46243 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dca2e0512a3240148025de7094376231 2024-11-27T17:39:08,876 DEBUG [M:0;8f0673442175:46243 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/31c4a30455684a728255d2fd751d9ebd is 812, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732729079862/Put/seqid=0 2024-11-27T17:39:08,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741870_1046 (size=14069) 2024-11-27T17:39:09,284 INFO [M:0;8f0673442175:46243 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=162.44 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/31c4a30455684a728255d2fd751d9ebd 2024-11-27T17:39:09,297 INFO [M:0;8f0673442175:46243 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 31c4a30455684a728255d2fd751d9ebd 2024-11-27T17:39:09,314 DEBUG [M:0;8f0673442175:46243 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3bf9a0abd49d44b1a92d750f475cd04d is 69, key is 8f0673442175,43899,1732729014292/rs:state/1732729047406/Put/seqid=0 2024-11-27T17:39:09,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741871_1047 (size=5156) 2024-11-27T17:39:09,721 INFO [M:0;8f0673442175:46243 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3bf9a0abd49d44b1a92d750f475cd04d 2024-11-27T17:39:09,735 DEBUG [M:0;8f0673442175:46243 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/dca2e0512a3240148025de7094376231 as hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/dca2e0512a3240148025de7094376231 2024-11-27T17:39:09,742 INFO [M:0;8f0673442175:46243 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/dca2e0512a3240148025de7094376231, entries=8, sequenceid=317, filesize=5.5 K 2024-11-27T17:39:09,743 DEBUG [M:0;8f0673442175:46243 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/31c4a30455684a728255d2fd751d9ebd as hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/31c4a30455684a728255d2fd751d9ebd 2024-11-27T17:39:09,750 INFO [M:0;8f0673442175:46243 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 31c4a30455684a728255d2fd751d9ebd 2024-11-27T17:39:09,750 INFO [M:0;8f0673442175:46243 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/31c4a30455684a728255d2fd751d9ebd, entries=33, sequenceid=317, filesize=13.7 K 2024-11-27T17:39:09,751 DEBUG [M:0;8f0673442175:46243 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3bf9a0abd49d44b1a92d750f475cd04d as hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3bf9a0abd49d44b1a92d750f475cd04d 2024-11-27T17:39:09,757 INFO [M:0;8f0673442175:46243 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40815/user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3bf9a0abd49d44b1a92d750f475cd04d, entries=1, sequenceid=317, filesize=5.0 K 2024-11-27T17:39:09,759 INFO [M:0;8f0673442175:46243 {}] regionserver.HRegion(3140): Finished flush of dataSize ~162.99 KB/166903, heapSize ~190.63 KB/195200, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1344ms, sequenceid=317, compaction requested=false 2024-11-27T17:39:09,760 INFO [M:0;8f0673442175:46243 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T17:39:09,760 DEBUG [M:0;8f0673442175:46243 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732729148413Disabling compacts and flushes for region at 1732729148413Disabling writes for close at 1732729148414 (+1 ms)Obtaining lock to block concurrent updates at 1732729148414Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732729148415 (+1 ms)Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=166903, getHeapSize=195440, getOffHeapSize=0, getCellsCount=364 at 1732729148416 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732729148417 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732729148418 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732729148435 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732729148435Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732729148857 (+422 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732729148875 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732729148875Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732729149297 (+422 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732729149313 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732729149313Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79fce99: reopening flushed file at 1732729149733 (+420 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d64dd80: reopening flushed file at 1732729149742 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b72e9df: reopening flushed file at 1732729149750 (+8 ms)Finished flush of dataSize ~162.99 KB/166903, heapSize ~190.63 KB/195200, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1344ms, sequenceid=317, compaction requested=false at 1732729149759 (+9 ms)Writing region close event to WAL at 1732729149760 (+1 ms)Closed at 1732729149760 2024-11-27T17:39:09,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:32923 is added to blk_1073741830_1006 (size=188115) 2024-11-27T17:39:09,764 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/79763024-49bb-e849-e376-9685aa12272f/MasterData/WALs/8f0673442175,46243,1732729013300/8f0673442175%2C46243%2C1732729013300.1732729016607 not finished, retry = 0 2024-11-27T17:39:09,836 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T17:39:09,865 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T17:39:09,865 INFO [M:0;8f0673442175:46243 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-27T17:39:09,865 INFO [M:0;8f0673442175:46243 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46243 2024-11-27T17:39:09,865 INFO [M:0;8f0673442175:46243 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T17:39:10,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46243-0x1017d5e7cd30000, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T17:39:10,012 INFO [M:0;8f0673442175:46243 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T17:39:10,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46243-0x1017d5e7cd30000, quorum=127.0.0.1:64480, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T17:39:10,019 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5976fb41{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T17:39:10,022 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@8846ab8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T17:39:10,022 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T17:39:10,022 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3ef1caaa{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T17:39:10,022 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28c7ffe1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/hadoop.log.dir/,STOPPED} 2024-11-27T17:39:10,024 WARN [BP-358449881-172.17.0.3-1732729009514 heartbeating to localhost/127.0.0.1:40815 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-27T17:39:10,024 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-27T17:39:10,024 WARN [BP-358449881-172.17.0.3-1732729009514 heartbeating to localhost/127.0.0.1:40815 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-358449881-172.17.0.3-1732729009514 (Datanode Uuid 4bfa520b-375c-403e-933e-34e2b075d46f) service to localhost/127.0.0.1:40815 2024-11-27T17:39:10,024 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-27T17:39:10,025 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/cluster_6034e5b1-ec67-3510-6ba0-128f6811cfb0/data/data1/current/BP-358449881-172.17.0.3-1732729009514 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T17:39:10,026 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/cluster_6034e5b1-ec67-3510-6ba0-128f6811cfb0/data/data2/current/BP-358449881-172.17.0.3-1732729009514 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T17:39:10,026 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-27T17:39:10,033 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@eb30bc1{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T17:39:10,034 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d80b5d0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T17:39:10,034 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T17:39:10,034 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@b7aa261{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T17:39:10,034 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1868869a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/a8f9e082-0e8d-eece-71cb-55a0b6cca68c/hadoop.log.dir/,STOPPED} 2024-11-27T17:39:10,040 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-27T17:39:10,063 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down